# from tvm.script import ir as I # from tvm.script import tir as T # from tvm.script import relax as R @I.ir_module class Module: I.module_attrs({"system_lib_prefix": "gemma_q0f16_"}) @T.prim_func def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") # with T.block("root"): for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) T.writes(logits[seq_ids[vs], vv]) logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) @T.prim_func def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") logit_bias = T.match_buffer(var_logit_bias, (num_token,)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) T.writes(logits[pos2seq_id[vp], token_ids[vp]]) logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] @T.prim_func def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") penalties = T.match_buffer(var_penalties, (num_seq, 3)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) @T.prim_func def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 256), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 256), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.090168440055560212) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): for tz in T.thread_binding(1, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz * 4 + ty, tx * 4 - 128:tx * 4 - 128 + 260]) T.writes(output[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty, tx * 4:tx * 4 + 4], lse[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((4, 256), "float16", scope="shared") V_smem = T.alloc_buffer((4, 256), "float16", scope="shared") O_allreduce = T.alloc_buffer((1, 4, 256), scope="shared") md_allreduce = T.alloc_buffer((1, 4, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((4,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = 0 bz: T.int32 = fused_by_bz batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec + 128] * T.float16(-1.0), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 3) // 4): tile_start_s: T.int32 = tz * 4 + ty tile_start_g: T.int32 = (iterator + tz) * 4 + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = row_g page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, pages[page_no, 0, by, page_offset, tx * 4 + vec + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(4): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz * 4 + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if (iterator + tz) * 4 + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(4): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(4): for vec in T.vectorized(4): V_local[vec] = V_smem[tz * 4 + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by * 8 + bz * 4 + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by * 8 + bz * 4 + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 256), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 256), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.090168440055560212) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): for tz in T.thread_binding(1, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz * 4 + ty, tx * 4 - 128:tx * 4 - 128 + 260]) T.writes(output[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty, tx * 4:tx * 4 + 4], lse[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((4, 256), "float16", scope="shared") V_smem = T.alloc_buffer((4, 256), "float16", scope="shared") O_allreduce = T.alloc_buffer((1, 4, 256), scope="shared") md_allreduce = T.alloc_buffer((1, 4, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((4,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = 0 bz: T.int32 = fused_by_bz batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec + 128] * T.float16(-1.0), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 3) // 4): tile_start_s: T.int32 = tz * 4 + ty tile_start_g: T.int32 = (iterator + tz) * 4 + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, pages[page_no, 0, by, page_offset, tx * 4 + vec + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(4): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz * 4 + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if (iterator + tz) * 4 + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(4): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(4): for vec in T.vectorized(4): V_local[vec] = V_smem[tz * 4 + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by * 8 + bz * 4 + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by * 8 + bz * 4 + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((16, 256), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, pages[page_no, 0, by, page_offset, j + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:16, 0:256]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(1, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 1, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(1, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((16, 256), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, pages[page_no, 0, by, page_offset, j + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:16, 0:256]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(1, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 1, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(1, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 1, 256), "float16") v = T.match_buffer(var_v, (kv_len, 1, 256), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(64, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((256, 16), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] q_indptr_val: T.int32 = q_indptr[b_idx] LH_start: T.int32 = tile_id[0] * 16 kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_0, lj_0_0 in T.grid(2, 2): for lj_1 in T.vectorized(8): with T.block("Q_load"): i = T.axis.spatial(16, li_0 * 8 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 16) j = T.axis.spatial(256, lj_0_0 * 128 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 16 * 8 + lj_1) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(2, 2): for ly_1 in T.vectorized(8): with T.block("K_load"): i = T.axis.spatial(16, lz_0 * 8 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 16) j = T.axis.spatial(256, ly_0_0 * 128 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 16 * 8 + ly_1) T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 128:j - 128 + 257]) T.writes(K_smem[j, i]) cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: freq = T.float32() K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, k[L_kv_base + cur_L, by, j + 128] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), k[L_kv_base + cur_L, by, j]) else: K_smem[j, i] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(2, 2): for ly_1 in T.vectorized(8): with T.block("V_load"): i = T.axis.spatial(16, lz_0 * 8 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 16) j = T.axis.spatial(256, ly_0_0 * 128 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 16 * 8 + ly_1) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: V_smem[i, j] = v[L_kv_base + cur_L, by, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:256, 0:16]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(1): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_0_init * 2 + lj_1_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0 in range(16): for li_1 in T.unroll(1): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(2): for lk_1 in range(16): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1_0 * 2 + lj_1_1) k_1 = T.axis.reduce(256, lk_0 * 16 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(1): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1_0 * 2 + lj_1_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(4): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_0_init * 8 + lj_1_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1 in T.grid(1, 16): for li_1 in T.unroll(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) k_1 = T.axis.reduce(16, lk_0 * 16 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 64 @T.prim_func def batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 256), "float16") q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 1, 256), "float16") v = T.match_buffer(var_v, (kv_len, 1, 256), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) tree_size = T.int32(is_size_var=True) mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((32, 256), "float16", scope="shared") V_smem = T.alloc_buffer((32, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 32), scope="shared") S_local = T.alloc_buffer((16, 32), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("KV_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_base + L_kv_start + i if L_kv_start + i < kv_chunk_len[0]: freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, k[cur_L, by, j + 128] * T.float16(-1.0), k[cur_L, by, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), k[cur_L, by, j]) V_smem[i, j] = v[cur_L, by, j] else: K_smem[i, j] = T.float16(0.0) V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:32, 0:256]) T.writes(S_local[0:16, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) k_1 = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 16: row_: T.int32 = (LH_start + row) // 8 if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:32], V_smem[0:32, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int64(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) # with T.block("root"): A_pad = T.alloc_buffer((batch_size, num_chunks, T.int64(4096))) temp_max = T.alloc_buffer((batch_size, num_chunks)) temp_sum = T.alloc_buffer((batch_size, num_chunks)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("pad"): v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2]) T.writes(A_pad[v0, v1, v2]) A_pad[v0, v1, v2] = T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * T.int64(4096) + v2] / temperature[v0], A[v0, v1 * T.int64(4096) + v2]), T.float32(-340282346638528859811704183484516925440.0)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("max"): v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) T.reads(A_pad[v0, v1, v2]) T.writes(temp_max[v0, v1]) with T.init(): temp_max[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) temp_max[v0, v1] = T.max(temp_max[v0, v1], A_pad[v0, v1, v2]) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("sum_exp"): v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) T.reads(temperature[v0], A_pad[v0, v1, v2], temp_max[v0, v1]) T.writes(temp_sum[v0, v1]) with T.init(): temp_sum[v0, v1] = T.float32(0.0) temp_sum[v0, v1] = temp_sum[v0, v1] + T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A_pad[v0, v1, v2] - temp_max[v0, v1]), T.Cast("float32", A_pad[v0, v1, v2] == temp_max[v0, v1])), T.float32(0.0)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(1)): with T.block("log"): v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) T.reads(temperature[v0], temp_sum[v0, v1], temp_max[v0, v1]) T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum[v0, v1]), temp_sum[v0, v1]) chunked_max[v0, v1] = temp_max[v0, v1] @T.prim_func def compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 1, 16, 256), "float16", offset_factor=1) copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) total_copy_length = T.int32() copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) with T.block("root"): T.reads() T.writes() for bhd_o in T.thread_binding(batch_size, thread="blockIdx.x"): for bhd_i in T.thread_binding(256, thread="threadIdx.x"): b: T.int32 = (bhd_o * 256 + bhd_i) // 256 h: T.int32 = 0 d: T.int32 = (bhd_o * 256 + bhd_i) % 256 if bhd_o * 256 + bhd_i < batch_size * 256: for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] @T.prim_func def copy_single_page(var_pages: T.handle, src_page_id: T.int64, tgt_page_id: T.int64, copy_length: T.int64): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages, page_size = T.int32(), T.int64() pages = T.match_buffer(var_pages, (num_pages, 2, 1, page_size, 256), "float16", offset_factor=1) # with T.block("root"): for b in T.thread_binding(copy_length, thread="blockIdx.x"): for t in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy"): vh = T.axis.spatial(1, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) // (copy_length * T.int64(256)))) vp = T.axis.spatial(copy_length, (b * T.int64(256) + T.Cast("int64", t)) % (copy_length * T.int64(256)) // T.int64(256)) vd = T.axis.spatial(256, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) % T.int64(256))) T.where(b * T.int64(256) + T.Cast("int64", t) < copy_length * T.int64(256)) T.reads(pages[src_page_id, 0:2, vh, vp, vd]) T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] @T.prim_func def fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) seq_len = T.int32() qkv = T.match_buffer(var_qkv, (seq_len, 10, 256), "float16") position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) q = T.match_buffer(var_q, (seq_len, 8, 256), "float16") k = T.match_buffer(var_k, (seq_len, 1, 256), "float16") v = T.match_buffer(var_v, (seq_len, 1, 256), "float16") # with T.block("root"): for iters_0, iters_1, iters_2 in T.grid(seq_len, 10, 256): with T.block("llama_fused_rope"): s, h, d = T.axis.remap("SSS", [iters_0, iters_1, iters_2]) T.reads(position_map[s], qkv[s, h, d - 128:d - 128 + 257]) T.writes(q[s, h, d], k[s, h - 8, d], v[s, h - 9, d]) if h < 8: freq = T.float32() q[s, h, d] = T.if_then_else(apply_rope > 0 and d < 256, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 128, qkv[s, h, d + 128] * T.float16(-1.0), qkv[s, h, d - 128]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 256) / T.float32(256.0))}), qkv[s, h, d]) else: if h < 9: freq = T.float32() k[s, h - 8, d] = T.if_then_else(apply_rope > 0 and d < 256, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 128, qkv[s, h, d + 128] * T.float16(-1.0), qkv[s, h, d - 128]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 256) / T.float32(256.0))}), qkv[s, h, d]) else: v[s, h - 9, d] = qkv[s, h, d] @T.prim_func def gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (m, n)) batch_size = T.int32(is_size_var=True) indices = T.match_buffer(var_indices, (batch_size,), "int32") dst = T.match_buffer(var_dst, (batch_size, n)) # with T.block("root"): for b, j in T.grid(batch_size, n): with T.block("gather_2d"): vb, vj = T.axis.remap("SS", [b, j]) T.reads(src[indices[vb], vj], indices[vb]) T.writes(dst[vb, vj]) dst[vb, vj] = src[indices[vb], vj] @T.prim_func(private=True) def index(var_rms_norm36: T.handle, index: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) seq_len = T.int64() rms_norm36 = T.match_buffer(var_rms_norm36, (T.int64(1), seq_len, T.int64(2048)), "float16") # with T.block("root"): for i, _, k in T.grid(T.int64(1), T.int64(1), T.int64(2048)): with T.block("index"): v_i, v__, v_k = T.axis.remap("SSS", [i, _, k]) T.reads(rms_norm36[v_i, seq_len - T.int64(1), v_k]) T.writes(index[v_i, v__, v_k]) index[v_i, v__, v_k] = rms_norm36[v_i, seq_len - T.int64(1), v_k] @T.prim_func def merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) V = T.match_buffer(v, (N, H, D), "float16") S = T.match_buffer(s, (N, H)) V_other = T.match_buffer(v_other, (N, H, D), "float16") S_other = T.match_buffer(s_other, (N, H)) # with T.block("root"): for bx in T.thread_binding(N, thread="blockIdx.x"): for by in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): with T.block("merge"): T.reads(S[bx, ty + by * 4], S_other[bx, ty + by * 4], V[bx, ty + by * 4, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 4, tx * 4:tx * 4 + 4]) T.writes(V[bx, ty + by * 4, tx * 4:tx * 4 + 4], S[bx, ty + by * 4]) s_val = T.alloc_buffer((1,), scope="local") s_other_val = T.alloc_buffer((1,), scope="local") s_max = T.alloc_buffer((1,), scope="local") scale = T.alloc_buffer((1,), scope="local") other_scale = T.alloc_buffer((1,), scope="local") v_vec = T.alloc_buffer((4,), "float16", scope="local") v_other_vec = T.alloc_buffer((4,), "float16", scope="local") s_val[0] = S[bx, ty + by * 4] s_other_val[0] = S_other[bx, ty + by * 4] s_max[0] = T.max(s_val[0], s_other_val[0]) s_val[0] = T.exp2(s_val[0] - s_max[0]) s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) for vec in T.vectorized(4): v_vec[vec] = V[bx, ty + by * 4, tx * 4 + vec] for vec in T.vectorized(4): v_other_vec[vec] = V_other[bx, ty + by * 4, tx * 4 + vec] for vec in range(4): v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) for vec in T.vectorized(4): V[bx, ty + by * 4, tx * 4 + vec] = v_vec[vec] S[bx, ty + by * 4] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] @T.prim_func def scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (batch_size, n)) indices = T.match_buffer(var_indices, (batch_size,), "int32") m = T.int32(is_size_var=True) dst = T.match_buffer(var_dst, (m, n)) # with T.block("root"): for b, j in T.grid(batch_size, n): with T.block("scatter_2d"): vb, vj = T.axis.remap("SS", [b, j]) T.reads(src[vb, vj], indices[vb]) T.writes(dst[indices[vb], vj]) dst[indices[vb], vj] = src[vb, vj] @T.prim_func def softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int64(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) T.reads(chunked_max[v0, v1]) T.writes(temp_max_shared[v0]) with T.init(): temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) T.writes(temp_sum_shared[v0]) with T.init(): temp_sum_shared[v0] = T.float32(0.0) temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) for l2_0 in T.serial(T.int64(16), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): for l2_1 in T.thread_binding(T.int64(8), thread="threadIdx.y"): for l2_2 in T.thread_binding(T.int64(32), thread="threadIdx.x"): with T.block("log_pad"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) v2 = T.axis.spatial(T.int64(4096), l2_0 * T.int64(256) + l2_1 * T.int64(32) + l2_2) T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2], temp_sum_shared[v0], temp_max_shared[v0]) T.writes(softmax[v0, v1 * T.int64(4096) + v2]) if v1 * T.int64(4096) + v2 < vocab_size: softmax[v0, v1 * T.int64(4096) + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * T.int64(4096) + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * T.int64(4096) + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) @T.prim_func def tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int64): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) num_pages, page_size = T.int64(), T.int64(is_size_var=True) pages = T.match_buffer(var_pages, (num_pages, 2, 1, page_size, 256), "float16", offset_factor=1) seqlen = T.int64(is_size_var=True) position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) k_data = T.match_buffer(var_k_data, (18, seqlen, 1, 256), "float16") v_data = T.match_buffer(var_v_data, (18, seqlen, 1, 256), "float16") # with T.block("root"): for p, h, d in T.grid(seqlen, 1, 256): with T.block("copy0"): vp, vh, vd = T.axis.remap("SSS", [p, h, d]) T.reads(position_map[vp], pages[T.Cast("int64", position_map[vp]) // page_size, 0:2, vh, T.Cast("int64", position_map[vp]) % page_size, vd]) T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) position: T.int32 = position_map[vp] k_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 0, vh, T.Cast("int64", position) % page_size, vd] v_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 1, vh, T.Cast("int64", position) % page_size, vd] @T.prim_func def tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) num_pages = T.int64() pages = T.match_buffer(var_pages, (num_pages, 2, 1, 16, 256), "float16", offset_factor=1) ntoken = T.int64(is_size_var=True) k_data = T.match_buffer(var_k_data, (ntoken, 1, 256), "float16") v_data = T.match_buffer(var_v_data, (ntoken, 1, 256), "float16") position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) # with T.block("root"): for global_pos, h, f in T.grid(ntoken, 1, 256): if position_map[global_pos] != -1: with T.block("k_transpose_append"): vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] with T.block("v_transpose_append"): vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] @T.prim_func def tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 32, 256), "float16") page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) total_tree_order_len = T.int32(is_size_var=True) tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) # with T.block("root"): assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((32, 256), "float16", scope="shared") V_smem = T.alloc_buffer((32, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 32), scope="shared") S_local = T.alloc_buffer((16, 32), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 K_smem[i, j] = pages[page_no, 0, by, page_offset, j] else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:32, 0:256]) T.writes(S_local[0:16, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 16: row_: T.int32 = (LH_start + row) // 8 if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:32], V_smem[0:32, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @R.function def alloc_embedding_tensor() -> R.Tensor((8192, 2048), dtype="float16"): R.func_attr({"relax.memory_plan_dynamic_func_output": True}) gv: R.Tensor((8192, 2048), dtype="float16") = R.builtin.alloc_tensor(R.shape([8192, 2048]), R.dtype("float16"), R.prim_value(0), R.str("global")) return gv @R.function def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) with R.dataflow(): model_embed_tokens_weight4: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[109] mul57: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.multiply(input_embeds, R.const(45.25, "float16")) rms_norm111: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(mul57, model_layers_0_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims219: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight4, axes=None) matmul219: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm111, permute_dims219, out_dtype="void") reshape216: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul219, R.shape([batch_size, 1, 10, 256])) reshape217: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape216, R.shape([batch_size, 10, 256])) lv55 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape217), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape218: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv55, R.shape([batch_size, 1, 8, 256])) reshape219: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape218, R.shape([batch_size, 1, 2048])) permute_dims220: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight4, axes=None) matmul220: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape219, permute_dims220, out_dtype="void") add108: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul220, mul57) rms_norm112: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add108, model_layers_0_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims221: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight4, axes=None) matmul221: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm112, permute_dims221, out_dtype="void") split54: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul221, indices_or_sections=2, axis=-1) split_054: R.Tensor((batch_size, 1, 16384), dtype="float16") = split54[0] split_154: R.Tensor((batch_size, 1, 16384), dtype="float16") = split54[1] gelu54: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_054) mul58: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu54, split_154) permute_dims222: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight4, axes=None) matmul222: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul58, permute_dims222, out_dtype="void") add109: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul222, add108) rms_norm113: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add109, model_layers_1_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims223: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_1_self_attn_qkv_proj_weight4, axes=None) matmul223: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm113, permute_dims223, out_dtype="void") reshape220: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul223, R.shape([batch_size, 1, 10, 256])) reshape221: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape220, R.shape([batch_size, 10, 256])) lv56 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape221), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape222: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv56, R.shape([batch_size, 1, 8, 256])) reshape223: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape222, R.shape([batch_size, 1, 2048])) permute_dims224: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_1_self_attn_o_proj_weight4, axes=None) matmul224: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape223, permute_dims224, out_dtype="void") add110: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul224, add109) rms_norm114: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add110, model_layers_1_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims225: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_1_mlp_gate_up_proj_weight4, axes=None) matmul225: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm114, permute_dims225, out_dtype="void") split55: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul225, indices_or_sections=2, axis=-1) split_055: R.Tensor((batch_size, 1, 16384), dtype="float16") = split55[0] split_155: R.Tensor((batch_size, 1, 16384), dtype="float16") = split55[1] gelu55: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_055) mul59: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu55, split_155) permute_dims226: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_1_mlp_down_proj_weight4, axes=None) matmul226: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul59, permute_dims226, out_dtype="void") add111: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul226, add110) rms_norm115: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add111, model_layers_2_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims227: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_2_self_attn_qkv_proj_weight4, axes=None) matmul227: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm115, permute_dims227, out_dtype="void") reshape224: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul227, R.shape([batch_size, 1, 10, 256])) reshape225: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape224, R.shape([batch_size, 10, 256])) lv57 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape225), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape226: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv57, R.shape([batch_size, 1, 8, 256])) reshape227: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape226, R.shape([batch_size, 1, 2048])) permute_dims228: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_2_self_attn_o_proj_weight4, axes=None) matmul228: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape227, permute_dims228, out_dtype="void") add112: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul228, add111) rms_norm116: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add112, model_layers_2_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims229: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_2_mlp_gate_up_proj_weight4, axes=None) matmul229: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm116, permute_dims229, out_dtype="void") split56: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul229, indices_or_sections=2, axis=-1) split_056: R.Tensor((batch_size, 1, 16384), dtype="float16") = split56[0] split_156: R.Tensor((batch_size, 1, 16384), dtype="float16") = split56[1] gelu56: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_056) mul60: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu56, split_156) permute_dims230: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_2_mlp_down_proj_weight4, axes=None) matmul230: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul60, permute_dims230, out_dtype="void") add113: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul230, add112) rms_norm117: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add113, model_layers_3_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims231: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_3_self_attn_qkv_proj_weight4, axes=None) matmul231: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm117, permute_dims231, out_dtype="void") reshape228: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul231, R.shape([batch_size, 1, 10, 256])) reshape229: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape228, R.shape([batch_size, 10, 256])) lv58 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape229), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape230: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv58, R.shape([batch_size, 1, 8, 256])) reshape231: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape230, R.shape([batch_size, 1, 2048])) permute_dims232: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_3_self_attn_o_proj_weight4, axes=None) matmul232: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape231, permute_dims232, out_dtype="void") add114: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul232, add113) rms_norm118: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add114, model_layers_3_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims233: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_3_mlp_gate_up_proj_weight4, axes=None) matmul233: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm118, permute_dims233, out_dtype="void") split57: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul233, indices_or_sections=2, axis=-1) split_057: R.Tensor((batch_size, 1, 16384), dtype="float16") = split57[0] split_157: R.Tensor((batch_size, 1, 16384), dtype="float16") = split57[1] gelu57: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_057) mul61: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu57, split_157) permute_dims234: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_3_mlp_down_proj_weight4, axes=None) matmul234: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul61, permute_dims234, out_dtype="void") add115: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul234, add114) rms_norm119: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add115, model_layers_4_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims235: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_4_self_attn_qkv_proj_weight4, axes=None) matmul235: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm119, permute_dims235, out_dtype="void") reshape232: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul235, R.shape([batch_size, 1, 10, 256])) reshape233: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape232, R.shape([batch_size, 10, 256])) lv59 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape233), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape234: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv59, R.shape([batch_size, 1, 8, 256])) reshape235: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape234, R.shape([batch_size, 1, 2048])) permute_dims236: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_4_self_attn_o_proj_weight4, axes=None) matmul236: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape235, permute_dims236, out_dtype="void") add116: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul236, add115) rms_norm120: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add116, model_layers_4_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims237: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_4_mlp_gate_up_proj_weight4, axes=None) matmul237: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm120, permute_dims237, out_dtype="void") split58: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul237, indices_or_sections=2, axis=-1) split_058: R.Tensor((batch_size, 1, 16384), dtype="float16") = split58[0] split_158: R.Tensor((batch_size, 1, 16384), dtype="float16") = split58[1] gelu58: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_058) mul62: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu58, split_158) permute_dims238: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_4_mlp_down_proj_weight4, axes=None) matmul238: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul62, permute_dims238, out_dtype="void") add117: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul238, add116) rms_norm121: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add117, model_layers_5_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims239: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_5_self_attn_qkv_proj_weight4, axes=None) matmul239: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm121, permute_dims239, out_dtype="void") reshape236: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul239, R.shape([batch_size, 1, 10, 256])) reshape237: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape236, R.shape([batch_size, 10, 256])) lv60 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape237), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape238: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv60, R.shape([batch_size, 1, 8, 256])) reshape239: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape238, R.shape([batch_size, 1, 2048])) permute_dims240: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_5_self_attn_o_proj_weight4, axes=None) matmul240: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape239, permute_dims240, out_dtype="void") add118: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul240, add117) rms_norm122: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add118, model_layers_5_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims241: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_5_mlp_gate_up_proj_weight4, axes=None) matmul241: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm122, permute_dims241, out_dtype="void") split59: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul241, indices_or_sections=2, axis=-1) split_059: R.Tensor((batch_size, 1, 16384), dtype="float16") = split59[0] split_159: R.Tensor((batch_size, 1, 16384), dtype="float16") = split59[1] gelu59: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_059) mul63: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu59, split_159) permute_dims242: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_5_mlp_down_proj_weight4, axes=None) matmul242: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul63, permute_dims242, out_dtype="void") add119: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul242, add118) rms_norm123: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add119, model_layers_6_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims243: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_6_self_attn_qkv_proj_weight4, axes=None) matmul243: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm123, permute_dims243, out_dtype="void") reshape240: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul243, R.shape([batch_size, 1, 10, 256])) reshape241: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape240, R.shape([batch_size, 10, 256])) lv61 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape241), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape242: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv61, R.shape([batch_size, 1, 8, 256])) reshape243: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape242, R.shape([batch_size, 1, 2048])) permute_dims244: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_6_self_attn_o_proj_weight4, axes=None) matmul244: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape243, permute_dims244, out_dtype="void") add120: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul244, add119) rms_norm124: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add120, model_layers_6_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims245: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_6_mlp_gate_up_proj_weight4, axes=None) matmul245: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm124, permute_dims245, out_dtype="void") split60: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul245, indices_or_sections=2, axis=-1) split_060: R.Tensor((batch_size, 1, 16384), dtype="float16") = split60[0] split_160: R.Tensor((batch_size, 1, 16384), dtype="float16") = split60[1] gelu60: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_060) mul64: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu60, split_160) permute_dims246: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_6_mlp_down_proj_weight4, axes=None) matmul246: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul64, permute_dims246, out_dtype="void") add121: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul246, add120) rms_norm125: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add121, model_layers_7_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims247: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_7_self_attn_qkv_proj_weight4, axes=None) matmul247: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm125, permute_dims247, out_dtype="void") reshape244: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul247, R.shape([batch_size, 1, 10, 256])) reshape245: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape244, R.shape([batch_size, 10, 256])) lv62 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape245), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape246: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv62, R.shape([batch_size, 1, 8, 256])) reshape247: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape246, R.shape([batch_size, 1, 2048])) permute_dims248: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_7_self_attn_o_proj_weight4, axes=None) matmul248: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape247, permute_dims248, out_dtype="void") add122: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul248, add121) rms_norm126: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add122, model_layers_7_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims249: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_7_mlp_gate_up_proj_weight4, axes=None) matmul249: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm126, permute_dims249, out_dtype="void") split61: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul249, indices_or_sections=2, axis=-1) split_061: R.Tensor((batch_size, 1, 16384), dtype="float16") = split61[0] split_161: R.Tensor((batch_size, 1, 16384), dtype="float16") = split61[1] gelu61: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_061) mul65: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu61, split_161) permute_dims250: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_7_mlp_down_proj_weight4, axes=None) matmul250: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul65, permute_dims250, out_dtype="void") add123: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul250, add122) rms_norm127: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add123, model_layers_8_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims251: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_8_self_attn_qkv_proj_weight4, axes=None) matmul251: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm127, permute_dims251, out_dtype="void") reshape248: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul251, R.shape([batch_size, 1, 10, 256])) reshape249: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape248, R.shape([batch_size, 10, 256])) lv63 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape249), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape250: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv63, R.shape([batch_size, 1, 8, 256])) reshape251: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape250, R.shape([batch_size, 1, 2048])) permute_dims252: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_8_self_attn_o_proj_weight4, axes=None) matmul252: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape251, permute_dims252, out_dtype="void") add124: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul252, add123) rms_norm128: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add124, model_layers_8_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims253: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_8_mlp_gate_up_proj_weight4, axes=None) matmul253: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm128, permute_dims253, out_dtype="void") split62: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul253, indices_or_sections=2, axis=-1) split_062: R.Tensor((batch_size, 1, 16384), dtype="float16") = split62[0] split_162: R.Tensor((batch_size, 1, 16384), dtype="float16") = split62[1] gelu62: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_062) mul66: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu62, split_162) permute_dims254: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_8_mlp_down_proj_weight4, axes=None) matmul254: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul66, permute_dims254, out_dtype="void") add125: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul254, add124) rms_norm129: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add125, model_layers_9_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims255: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_9_self_attn_qkv_proj_weight4, axes=None) matmul255: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm129, permute_dims255, out_dtype="void") reshape252: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul255, R.shape([batch_size, 1, 10, 256])) reshape253: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape252, R.shape([batch_size, 10, 256])) lv64 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape253), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape254: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv64, R.shape([batch_size, 1, 8, 256])) reshape255: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape254, R.shape([batch_size, 1, 2048])) permute_dims256: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_9_self_attn_o_proj_weight4, axes=None) matmul256: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape255, permute_dims256, out_dtype="void") add126: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul256, add125) rms_norm130: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add126, model_layers_9_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims257: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_9_mlp_gate_up_proj_weight4, axes=None) matmul257: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm130, permute_dims257, out_dtype="void") split63: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul257, indices_or_sections=2, axis=-1) split_063: R.Tensor((batch_size, 1, 16384), dtype="float16") = split63[0] split_163: R.Tensor((batch_size, 1, 16384), dtype="float16") = split63[1] gelu63: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_063) mul67: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu63, split_163) permute_dims258: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_9_mlp_down_proj_weight4, axes=None) matmul258: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul67, permute_dims258, out_dtype="void") add127: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul258, add126) rms_norm131: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add127, model_layers_10_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims259: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_10_self_attn_qkv_proj_weight4, axes=None) matmul259: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm131, permute_dims259, out_dtype="void") reshape256: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul259, R.shape([batch_size, 1, 10, 256])) reshape257: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape256, R.shape([batch_size, 10, 256])) lv65 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape257), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape258: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv65, R.shape([batch_size, 1, 8, 256])) reshape259: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape258, R.shape([batch_size, 1, 2048])) permute_dims260: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_10_self_attn_o_proj_weight4, axes=None) matmul260: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape259, permute_dims260, out_dtype="void") add128: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul260, add127) rms_norm132: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add128, model_layers_10_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims261: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_10_mlp_gate_up_proj_weight4, axes=None) matmul261: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm132, permute_dims261, out_dtype="void") split64: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul261, indices_or_sections=2, axis=-1) split_064: R.Tensor((batch_size, 1, 16384), dtype="float16") = split64[0] split_164: R.Tensor((batch_size, 1, 16384), dtype="float16") = split64[1] gelu64: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_064) mul68: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu64, split_164) permute_dims262: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_10_mlp_down_proj_weight4, axes=None) matmul262: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul68, permute_dims262, out_dtype="void") add129: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul262, add128) rms_norm133: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add129, model_layers_11_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims263: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_11_self_attn_qkv_proj_weight4, axes=None) matmul263: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm133, permute_dims263, out_dtype="void") reshape260: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul263, R.shape([batch_size, 1, 10, 256])) reshape261: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape260, R.shape([batch_size, 10, 256])) lv66 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape261), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape262: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv66, R.shape([batch_size, 1, 8, 256])) reshape263: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape262, R.shape([batch_size, 1, 2048])) permute_dims264: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_11_self_attn_o_proj_weight4, axes=None) matmul264: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape263, permute_dims264, out_dtype="void") add130: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul264, add129) rms_norm134: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add130, model_layers_11_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims265: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_11_mlp_gate_up_proj_weight4, axes=None) matmul265: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm134, permute_dims265, out_dtype="void") split65: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul265, indices_or_sections=2, axis=-1) split_065: R.Tensor((batch_size, 1, 16384), dtype="float16") = split65[0] split_165: R.Tensor((batch_size, 1, 16384), dtype="float16") = split65[1] gelu65: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_065) mul69: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu65, split_165) permute_dims266: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_11_mlp_down_proj_weight4, axes=None) matmul266: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul69, permute_dims266, out_dtype="void") add131: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul266, add130) rms_norm135: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add131, model_layers_12_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims267: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_12_self_attn_qkv_proj_weight4, axes=None) matmul267: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm135, permute_dims267, out_dtype="void") reshape264: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul267, R.shape([batch_size, 1, 10, 256])) reshape265: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape264, R.shape([batch_size, 10, 256])) lv67 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape265), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape266: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv67, R.shape([batch_size, 1, 8, 256])) reshape267: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape266, R.shape([batch_size, 1, 2048])) permute_dims268: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_12_self_attn_o_proj_weight4, axes=None) matmul268: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape267, permute_dims268, out_dtype="void") add132: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul268, add131) rms_norm136: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add132, model_layers_12_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims269: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_12_mlp_gate_up_proj_weight4, axes=None) matmul269: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm136, permute_dims269, out_dtype="void") split66: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul269, indices_or_sections=2, axis=-1) split_066: R.Tensor((batch_size, 1, 16384), dtype="float16") = split66[0] split_166: R.Tensor((batch_size, 1, 16384), dtype="float16") = split66[1] gelu66: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_066) mul70: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu66, split_166) permute_dims270: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_12_mlp_down_proj_weight4, axes=None) matmul270: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul70, permute_dims270, out_dtype="void") add133: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul270, add132) rms_norm137: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add133, model_layers_13_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims271: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_13_self_attn_qkv_proj_weight4, axes=None) matmul271: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm137, permute_dims271, out_dtype="void") reshape268: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul271, R.shape([batch_size, 1, 10, 256])) reshape269: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape268, R.shape([batch_size, 10, 256])) lv68 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape269), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape270: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv68, R.shape([batch_size, 1, 8, 256])) reshape271: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape270, R.shape([batch_size, 1, 2048])) permute_dims272: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_13_self_attn_o_proj_weight4, axes=None) matmul272: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape271, permute_dims272, out_dtype="void") add134: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul272, add133) rms_norm138: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add134, model_layers_13_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims273: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_13_mlp_gate_up_proj_weight4, axes=None) matmul273: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm138, permute_dims273, out_dtype="void") split67: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul273, indices_or_sections=2, axis=-1) split_067: R.Tensor((batch_size, 1, 16384), dtype="float16") = split67[0] split_167: R.Tensor((batch_size, 1, 16384), dtype="float16") = split67[1] gelu67: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_067) mul71: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu67, split_167) permute_dims274: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_13_mlp_down_proj_weight4, axes=None) matmul274: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul71, permute_dims274, out_dtype="void") add135: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul274, add134) rms_norm139: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add135, model_layers_14_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims275: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_14_self_attn_qkv_proj_weight4, axes=None) matmul275: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm139, permute_dims275, out_dtype="void") reshape272: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul275, R.shape([batch_size, 1, 10, 256])) reshape273: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape272, R.shape([batch_size, 10, 256])) lv69 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape273), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape274: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv69, R.shape([batch_size, 1, 8, 256])) reshape275: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape274, R.shape([batch_size, 1, 2048])) permute_dims276: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_14_self_attn_o_proj_weight4, axes=None) matmul276: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape275, permute_dims276, out_dtype="void") add136: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul276, add135) rms_norm140: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add136, model_layers_14_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims277: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_14_mlp_gate_up_proj_weight4, axes=None) matmul277: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm140, permute_dims277, out_dtype="void") split68: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul277, indices_or_sections=2, axis=-1) split_068: R.Tensor((batch_size, 1, 16384), dtype="float16") = split68[0] split_168: R.Tensor((batch_size, 1, 16384), dtype="float16") = split68[1] gelu68: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_068) mul72: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu68, split_168) permute_dims278: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_14_mlp_down_proj_weight4, axes=None) matmul278: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul72, permute_dims278, out_dtype="void") add137: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul278, add136) rms_norm141: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add137, model_layers_15_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims279: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_15_self_attn_qkv_proj_weight4, axes=None) matmul279: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm141, permute_dims279, out_dtype="void") reshape276: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul279, R.shape([batch_size, 1, 10, 256])) reshape277: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape276, R.shape([batch_size, 10, 256])) lv70 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape277), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape278: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv70, R.shape([batch_size, 1, 8, 256])) reshape279: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape278, R.shape([batch_size, 1, 2048])) permute_dims280: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_15_self_attn_o_proj_weight4, axes=None) matmul280: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape279, permute_dims280, out_dtype="void") add138: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul280, add137) rms_norm142: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add138, model_layers_15_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims281: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_15_mlp_gate_up_proj_weight4, axes=None) matmul281: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm142, permute_dims281, out_dtype="void") split69: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul281, indices_or_sections=2, axis=-1) split_069: R.Tensor((batch_size, 1, 16384), dtype="float16") = split69[0] split_169: R.Tensor((batch_size, 1, 16384), dtype="float16") = split69[1] gelu69: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_069) mul73: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu69, split_169) permute_dims282: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_15_mlp_down_proj_weight4, axes=None) matmul282: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul73, permute_dims282, out_dtype="void") add139: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul282, add138) rms_norm143: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add139, model_layers_16_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims283: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_16_self_attn_qkv_proj_weight4, axes=None) matmul283: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm143, permute_dims283, out_dtype="void") reshape280: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul283, R.shape([batch_size, 1, 10, 256])) reshape281: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape280, R.shape([batch_size, 10, 256])) lv71 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape281), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape282: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv71, R.shape([batch_size, 1, 8, 256])) reshape283: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape282, R.shape([batch_size, 1, 2048])) permute_dims284: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_16_self_attn_o_proj_weight4, axes=None) matmul284: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape283, permute_dims284, out_dtype="void") add140: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul284, add139) rms_norm144: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add140, model_layers_16_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims285: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_16_mlp_gate_up_proj_weight4, axes=None) matmul285: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm144, permute_dims285, out_dtype="void") split70: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul285, indices_or_sections=2, axis=-1) split_070: R.Tensor((batch_size, 1, 16384), dtype="float16") = split70[0] split_170: R.Tensor((batch_size, 1, 16384), dtype="float16") = split70[1] gelu70: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_070) mul74: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu70, split_170) permute_dims286: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_16_mlp_down_proj_weight4, axes=None) matmul286: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul74, permute_dims286, out_dtype="void") add141: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul286, add140) rms_norm145: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add141, model_layers_17_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims287: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_17_self_attn_qkv_proj_weight4, axes=None) matmul287: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm145, permute_dims287, out_dtype="void") reshape284: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(matmul287, R.shape([batch_size, 1, 10, 256])) reshape285: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape284, R.shape([batch_size, 10, 256])) lv72 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape285), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape286: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv72, R.shape([batch_size, 1, 8, 256])) reshape287: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape286, R.shape([batch_size, 1, 2048])) permute_dims288: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_17_self_attn_o_proj_weight4, axes=None) matmul288: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape287, permute_dims288, out_dtype="void") add142: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul288, add141) rms_norm146: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add142, model_layers_17_post_attention_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims289: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_17_mlp_gate_up_proj_weight4, axes=None) matmul289: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm146, permute_dims289, out_dtype="void") split71: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(matmul289, indices_or_sections=2, axis=-1) split_071: R.Tensor((batch_size, 1, 16384), dtype="float16") = split71[0] split_171: R.Tensor((batch_size, 1, 16384), dtype="float16") = split71[1] gelu71: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_071) mul75: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu71, split_171) permute_dims290: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_17_mlp_down_proj_weight4, axes=None) matmul290: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul75, permute_dims290, out_dtype="void") add143: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul290, add142) rms_norm147: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(add143, model_norm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims291: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight4, axes=None) matmul291: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.matmul(rms_norm147, permute_dims291, out_dtype="float32") gv4: R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype="float32"), R.Object) = matmul291, paged_kv_cache R.output(gv4) return gv4 @R.function def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) with R.dataflow(): model_embed_tokens_weight3: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[109] mul38: R.Tensor((1, seq_len, 2048), dtype="float16") = R.multiply(input_embeds, R.const(45.25, "float16")) rms_norm74: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(mul38, model_layers_0_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims146: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight3, axes=None) matmul146: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm74, permute_dims146, out_dtype="void") reshape144: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul146, R.shape([1, seq_len, 10, 256])) reshape145: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape144, R.shape([seq_len, 10, 256])) lv37 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape145), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape146: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv37, R.shape([1, seq_len, 8, 256])) reshape147: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape146, R.shape([1, seq_len, 2048])) permute_dims147: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight3, axes=None) matmul147: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape147, permute_dims147, out_dtype="void") add72: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul147, mul38) rms_norm75: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add72, model_layers_0_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims148: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight3, axes=None) matmul148: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm75, permute_dims148, out_dtype="void") split36: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul148, indices_or_sections=2, axis=-1) split_036: R.Tensor((1, seq_len, 16384), dtype="float16") = split36[0] split_136: R.Tensor((1, seq_len, 16384), dtype="float16") = split36[1] gelu36: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_036) mul39: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu36, split_136) permute_dims149: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight3, axes=None) matmul149: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul39, permute_dims149, out_dtype="void") add73: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul149, add72) rms_norm76: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add73, model_layers_1_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims150: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_1_self_attn_qkv_proj_weight3, axes=None) matmul150: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm76, permute_dims150, out_dtype="void") reshape148: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul150, R.shape([1, seq_len, 10, 256])) reshape149: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape148, R.shape([seq_len, 10, 256])) lv38 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape149), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape150: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv38, R.shape([1, seq_len, 8, 256])) reshape151: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape150, R.shape([1, seq_len, 2048])) permute_dims151: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_1_self_attn_o_proj_weight3, axes=None) matmul151: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape151, permute_dims151, out_dtype="void") add74: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul151, add73) rms_norm77: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add74, model_layers_1_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims152: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_1_mlp_gate_up_proj_weight3, axes=None) matmul152: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm77, permute_dims152, out_dtype="void") split37: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul152, indices_or_sections=2, axis=-1) split_037: R.Tensor((1, seq_len, 16384), dtype="float16") = split37[0] split_137: R.Tensor((1, seq_len, 16384), dtype="float16") = split37[1] gelu37: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_037) mul40: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu37, split_137) permute_dims153: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_1_mlp_down_proj_weight3, axes=None) matmul153: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul40, permute_dims153, out_dtype="void") add75: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul153, add74) rms_norm78: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add75, model_layers_2_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims154: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_2_self_attn_qkv_proj_weight3, axes=None) matmul154: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm78, permute_dims154, out_dtype="void") reshape152: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul154, R.shape([1, seq_len, 10, 256])) reshape153: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape152, R.shape([seq_len, 10, 256])) lv39 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape153), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape154: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv39, R.shape([1, seq_len, 8, 256])) reshape155: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape154, R.shape([1, seq_len, 2048])) permute_dims155: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_2_self_attn_o_proj_weight3, axes=None) matmul155: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape155, permute_dims155, out_dtype="void") add76: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul155, add75) rms_norm79: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add76, model_layers_2_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims156: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_2_mlp_gate_up_proj_weight3, axes=None) matmul156: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm79, permute_dims156, out_dtype="void") split38: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul156, indices_or_sections=2, axis=-1) split_038: R.Tensor((1, seq_len, 16384), dtype="float16") = split38[0] split_138: R.Tensor((1, seq_len, 16384), dtype="float16") = split38[1] gelu38: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_038) mul41: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu38, split_138) permute_dims157: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_2_mlp_down_proj_weight3, axes=None) matmul157: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul41, permute_dims157, out_dtype="void") add77: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul157, add76) rms_norm80: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add77, model_layers_3_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims158: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_3_self_attn_qkv_proj_weight3, axes=None) matmul158: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm80, permute_dims158, out_dtype="void") reshape156: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul158, R.shape([1, seq_len, 10, 256])) reshape157: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape156, R.shape([seq_len, 10, 256])) lv40 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape157), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape158: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv40, R.shape([1, seq_len, 8, 256])) reshape159: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape158, R.shape([1, seq_len, 2048])) permute_dims159: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_3_self_attn_o_proj_weight3, axes=None) matmul159: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape159, permute_dims159, out_dtype="void") add78: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul159, add77) rms_norm81: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add78, model_layers_3_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims160: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_3_mlp_gate_up_proj_weight3, axes=None) matmul160: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm81, permute_dims160, out_dtype="void") split39: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul160, indices_or_sections=2, axis=-1) split_039: R.Tensor((1, seq_len, 16384), dtype="float16") = split39[0] split_139: R.Tensor((1, seq_len, 16384), dtype="float16") = split39[1] gelu39: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_039) mul42: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu39, split_139) permute_dims161: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_3_mlp_down_proj_weight3, axes=None) matmul161: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul42, permute_dims161, out_dtype="void") add79: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul161, add78) rms_norm82: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add79, model_layers_4_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims162: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_4_self_attn_qkv_proj_weight3, axes=None) matmul162: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm82, permute_dims162, out_dtype="void") reshape160: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul162, R.shape([1, seq_len, 10, 256])) reshape161: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape160, R.shape([seq_len, 10, 256])) lv41 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape161), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape162: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv41, R.shape([1, seq_len, 8, 256])) reshape163: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape162, R.shape([1, seq_len, 2048])) permute_dims163: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_4_self_attn_o_proj_weight3, axes=None) matmul163: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape163, permute_dims163, out_dtype="void") add80: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul163, add79) rms_norm83: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add80, model_layers_4_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims164: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_4_mlp_gate_up_proj_weight3, axes=None) matmul164: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm83, permute_dims164, out_dtype="void") split40: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul164, indices_or_sections=2, axis=-1) split_040: R.Tensor((1, seq_len, 16384), dtype="float16") = split40[0] split_140: R.Tensor((1, seq_len, 16384), dtype="float16") = split40[1] gelu40: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_040) mul43: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu40, split_140) permute_dims165: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_4_mlp_down_proj_weight3, axes=None) matmul165: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul43, permute_dims165, out_dtype="void") add81: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul165, add80) rms_norm84: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add81, model_layers_5_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims166: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_5_self_attn_qkv_proj_weight3, axes=None) matmul166: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm84, permute_dims166, out_dtype="void") reshape164: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul166, R.shape([1, seq_len, 10, 256])) reshape165: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape164, R.shape([seq_len, 10, 256])) lv42 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape165), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape166: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv42, R.shape([1, seq_len, 8, 256])) reshape167: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape166, R.shape([1, seq_len, 2048])) permute_dims167: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_5_self_attn_o_proj_weight3, axes=None) matmul167: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape167, permute_dims167, out_dtype="void") add82: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul167, add81) rms_norm85: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add82, model_layers_5_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims168: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_5_mlp_gate_up_proj_weight3, axes=None) matmul168: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm85, permute_dims168, out_dtype="void") split41: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul168, indices_or_sections=2, axis=-1) split_041: R.Tensor((1, seq_len, 16384), dtype="float16") = split41[0] split_141: R.Tensor((1, seq_len, 16384), dtype="float16") = split41[1] gelu41: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_041) mul44: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu41, split_141) permute_dims169: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_5_mlp_down_proj_weight3, axes=None) matmul169: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul44, permute_dims169, out_dtype="void") add83: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul169, add82) rms_norm86: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add83, model_layers_6_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims170: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_6_self_attn_qkv_proj_weight3, axes=None) matmul170: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm86, permute_dims170, out_dtype="void") reshape168: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul170, R.shape([1, seq_len, 10, 256])) reshape169: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape168, R.shape([seq_len, 10, 256])) lv43 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape169), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape170: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv43, R.shape([1, seq_len, 8, 256])) reshape171: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape170, R.shape([1, seq_len, 2048])) permute_dims171: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_6_self_attn_o_proj_weight3, axes=None) matmul171: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape171, permute_dims171, out_dtype="void") add84: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul171, add83) rms_norm87: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add84, model_layers_6_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims172: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_6_mlp_gate_up_proj_weight3, axes=None) matmul172: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm87, permute_dims172, out_dtype="void") split42: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul172, indices_or_sections=2, axis=-1) split_042: R.Tensor((1, seq_len, 16384), dtype="float16") = split42[0] split_142: R.Tensor((1, seq_len, 16384), dtype="float16") = split42[1] gelu42: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_042) mul45: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu42, split_142) permute_dims173: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_6_mlp_down_proj_weight3, axes=None) matmul173: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul45, permute_dims173, out_dtype="void") add85: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul173, add84) rms_norm88: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add85, model_layers_7_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims174: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_7_self_attn_qkv_proj_weight3, axes=None) matmul174: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm88, permute_dims174, out_dtype="void") reshape172: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul174, R.shape([1, seq_len, 10, 256])) reshape173: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape172, R.shape([seq_len, 10, 256])) lv44 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape173), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape174: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv44, R.shape([1, seq_len, 8, 256])) reshape175: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape174, R.shape([1, seq_len, 2048])) permute_dims175: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_7_self_attn_o_proj_weight3, axes=None) matmul175: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape175, permute_dims175, out_dtype="void") add86: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul175, add85) rms_norm89: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add86, model_layers_7_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims176: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_7_mlp_gate_up_proj_weight3, axes=None) matmul176: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm89, permute_dims176, out_dtype="void") split43: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul176, indices_or_sections=2, axis=-1) split_043: R.Tensor((1, seq_len, 16384), dtype="float16") = split43[0] split_143: R.Tensor((1, seq_len, 16384), dtype="float16") = split43[1] gelu43: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_043) mul46: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu43, split_143) permute_dims177: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_7_mlp_down_proj_weight3, axes=None) matmul177: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul46, permute_dims177, out_dtype="void") add87: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul177, add86) rms_norm90: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add87, model_layers_8_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims178: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_8_self_attn_qkv_proj_weight3, axes=None) matmul178: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm90, permute_dims178, out_dtype="void") reshape176: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul178, R.shape([1, seq_len, 10, 256])) reshape177: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape176, R.shape([seq_len, 10, 256])) lv45 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape177), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape178: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv45, R.shape([1, seq_len, 8, 256])) reshape179: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape178, R.shape([1, seq_len, 2048])) permute_dims179: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_8_self_attn_o_proj_weight3, axes=None) matmul179: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape179, permute_dims179, out_dtype="void") add88: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul179, add87) rms_norm91: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add88, model_layers_8_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims180: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_8_mlp_gate_up_proj_weight3, axes=None) matmul180: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm91, permute_dims180, out_dtype="void") split44: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul180, indices_or_sections=2, axis=-1) split_044: R.Tensor((1, seq_len, 16384), dtype="float16") = split44[0] split_144: R.Tensor((1, seq_len, 16384), dtype="float16") = split44[1] gelu44: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_044) mul47: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu44, split_144) permute_dims181: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_8_mlp_down_proj_weight3, axes=None) matmul181: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul47, permute_dims181, out_dtype="void") add89: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul181, add88) rms_norm92: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add89, model_layers_9_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims182: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_9_self_attn_qkv_proj_weight3, axes=None) matmul182: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm92, permute_dims182, out_dtype="void") reshape180: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul182, R.shape([1, seq_len, 10, 256])) reshape181: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape180, R.shape([seq_len, 10, 256])) lv46 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape181), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape182: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv46, R.shape([1, seq_len, 8, 256])) reshape183: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape182, R.shape([1, seq_len, 2048])) permute_dims183: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_9_self_attn_o_proj_weight3, axes=None) matmul183: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape183, permute_dims183, out_dtype="void") add90: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul183, add89) rms_norm93: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add90, model_layers_9_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims184: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_9_mlp_gate_up_proj_weight3, axes=None) matmul184: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm93, permute_dims184, out_dtype="void") split45: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul184, indices_or_sections=2, axis=-1) split_045: R.Tensor((1, seq_len, 16384), dtype="float16") = split45[0] split_145: R.Tensor((1, seq_len, 16384), dtype="float16") = split45[1] gelu45: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_045) mul48: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu45, split_145) permute_dims185: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_9_mlp_down_proj_weight3, axes=None) matmul185: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul48, permute_dims185, out_dtype="void") add91: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul185, add90) rms_norm94: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add91, model_layers_10_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims186: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_10_self_attn_qkv_proj_weight3, axes=None) matmul186: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm94, permute_dims186, out_dtype="void") reshape184: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul186, R.shape([1, seq_len, 10, 256])) reshape185: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape184, R.shape([seq_len, 10, 256])) lv47 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape185), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape186: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv47, R.shape([1, seq_len, 8, 256])) reshape187: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape186, R.shape([1, seq_len, 2048])) permute_dims187: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_10_self_attn_o_proj_weight3, axes=None) matmul187: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape187, permute_dims187, out_dtype="void") add92: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul187, add91) rms_norm95: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add92, model_layers_10_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims188: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_10_mlp_gate_up_proj_weight3, axes=None) matmul188: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm95, permute_dims188, out_dtype="void") split46: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul188, indices_or_sections=2, axis=-1) split_046: R.Tensor((1, seq_len, 16384), dtype="float16") = split46[0] split_146: R.Tensor((1, seq_len, 16384), dtype="float16") = split46[1] gelu46: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_046) mul49: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu46, split_146) permute_dims189: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_10_mlp_down_proj_weight3, axes=None) matmul189: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul49, permute_dims189, out_dtype="void") add93: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul189, add92) rms_norm96: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add93, model_layers_11_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims190: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_11_self_attn_qkv_proj_weight3, axes=None) matmul190: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm96, permute_dims190, out_dtype="void") reshape188: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul190, R.shape([1, seq_len, 10, 256])) reshape189: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape188, R.shape([seq_len, 10, 256])) lv48 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape189), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape190: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv48, R.shape([1, seq_len, 8, 256])) reshape191: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape190, R.shape([1, seq_len, 2048])) permute_dims191: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_11_self_attn_o_proj_weight3, axes=None) matmul191: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape191, permute_dims191, out_dtype="void") add94: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul191, add93) rms_norm97: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add94, model_layers_11_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims192: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_11_mlp_gate_up_proj_weight3, axes=None) matmul192: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm97, permute_dims192, out_dtype="void") split47: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul192, indices_or_sections=2, axis=-1) split_047: R.Tensor((1, seq_len, 16384), dtype="float16") = split47[0] split_147: R.Tensor((1, seq_len, 16384), dtype="float16") = split47[1] gelu47: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_047) mul50: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu47, split_147) permute_dims193: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_11_mlp_down_proj_weight3, axes=None) matmul193: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul50, permute_dims193, out_dtype="void") add95: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul193, add94) rms_norm98: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add95, model_layers_12_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims194: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_12_self_attn_qkv_proj_weight3, axes=None) matmul194: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm98, permute_dims194, out_dtype="void") reshape192: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul194, R.shape([1, seq_len, 10, 256])) reshape193: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape192, R.shape([seq_len, 10, 256])) lv49 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape193), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape194: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv49, R.shape([1, seq_len, 8, 256])) reshape195: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape194, R.shape([1, seq_len, 2048])) permute_dims195: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_12_self_attn_o_proj_weight3, axes=None) matmul195: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape195, permute_dims195, out_dtype="void") add96: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul195, add95) rms_norm99: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add96, model_layers_12_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims196: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_12_mlp_gate_up_proj_weight3, axes=None) matmul196: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm99, permute_dims196, out_dtype="void") split48: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul196, indices_or_sections=2, axis=-1) split_048: R.Tensor((1, seq_len, 16384), dtype="float16") = split48[0] split_148: R.Tensor((1, seq_len, 16384), dtype="float16") = split48[1] gelu48: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_048) mul51: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu48, split_148) permute_dims197: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_12_mlp_down_proj_weight3, axes=None) matmul197: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul51, permute_dims197, out_dtype="void") add97: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul197, add96) rms_norm100: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add97, model_layers_13_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims198: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_13_self_attn_qkv_proj_weight3, axes=None) matmul198: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm100, permute_dims198, out_dtype="void") reshape196: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul198, R.shape([1, seq_len, 10, 256])) reshape197: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape196, R.shape([seq_len, 10, 256])) lv50 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape197), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape198: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv50, R.shape([1, seq_len, 8, 256])) reshape199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape198, R.shape([1, seq_len, 2048])) permute_dims199: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_13_self_attn_o_proj_weight3, axes=None) matmul199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape199, permute_dims199, out_dtype="void") add98: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul199, add97) rms_norm101: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add98, model_layers_13_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims200: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_13_mlp_gate_up_proj_weight3, axes=None) matmul200: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm101, permute_dims200, out_dtype="void") split49: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul200, indices_or_sections=2, axis=-1) split_049: R.Tensor((1, seq_len, 16384), dtype="float16") = split49[0] split_149: R.Tensor((1, seq_len, 16384), dtype="float16") = split49[1] gelu49: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_049) mul52: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu49, split_149) permute_dims201: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_13_mlp_down_proj_weight3, axes=None) matmul201: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul52, permute_dims201, out_dtype="void") add99: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul201, add98) rms_norm102: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add99, model_layers_14_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims202: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_14_self_attn_qkv_proj_weight3, axes=None) matmul202: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm102, permute_dims202, out_dtype="void") reshape200: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul202, R.shape([1, seq_len, 10, 256])) reshape201: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape200, R.shape([seq_len, 10, 256])) lv51 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape201), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape202: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv51, R.shape([1, seq_len, 8, 256])) reshape203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape202, R.shape([1, seq_len, 2048])) permute_dims203: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_14_self_attn_o_proj_weight3, axes=None) matmul203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape203, permute_dims203, out_dtype="void") add100: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul203, add99) rms_norm103: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add100, model_layers_14_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims204: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_14_mlp_gate_up_proj_weight3, axes=None) matmul204: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm103, permute_dims204, out_dtype="void") split50: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul204, indices_or_sections=2, axis=-1) split_050: R.Tensor((1, seq_len, 16384), dtype="float16") = split50[0] split_150: R.Tensor((1, seq_len, 16384), dtype="float16") = split50[1] gelu50: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_050) mul53: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu50, split_150) permute_dims205: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_14_mlp_down_proj_weight3, axes=None) matmul205: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul53, permute_dims205, out_dtype="void") add101: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul205, add100) rms_norm104: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add101, model_layers_15_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims206: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_15_self_attn_qkv_proj_weight3, axes=None) matmul206: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm104, permute_dims206, out_dtype="void") reshape204: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul206, R.shape([1, seq_len, 10, 256])) reshape205: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape204, R.shape([seq_len, 10, 256])) lv52 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape205), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape206: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv52, R.shape([1, seq_len, 8, 256])) reshape207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape206, R.shape([1, seq_len, 2048])) permute_dims207: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_15_self_attn_o_proj_weight3, axes=None) matmul207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape207, permute_dims207, out_dtype="void") add102: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul207, add101) rms_norm105: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add102, model_layers_15_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims208: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_15_mlp_gate_up_proj_weight3, axes=None) matmul208: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm105, permute_dims208, out_dtype="void") split51: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul208, indices_or_sections=2, axis=-1) split_051: R.Tensor((1, seq_len, 16384), dtype="float16") = split51[0] split_151: R.Tensor((1, seq_len, 16384), dtype="float16") = split51[1] gelu51: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_051) mul54: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu51, split_151) permute_dims209: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_15_mlp_down_proj_weight3, axes=None) matmul209: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul54, permute_dims209, out_dtype="void") add103: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul209, add102) rms_norm106: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add103, model_layers_16_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims210: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_16_self_attn_qkv_proj_weight3, axes=None) matmul210: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm106, permute_dims210, out_dtype="void") reshape208: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul210, R.shape([1, seq_len, 10, 256])) reshape209: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape208, R.shape([seq_len, 10, 256])) lv53 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape209), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape210: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv53, R.shape([1, seq_len, 8, 256])) reshape211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape210, R.shape([1, seq_len, 2048])) permute_dims211: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_16_self_attn_o_proj_weight3, axes=None) matmul211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape211, permute_dims211, out_dtype="void") add104: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul211, add103) rms_norm107: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add104, model_layers_16_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims212: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_16_mlp_gate_up_proj_weight3, axes=None) matmul212: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm107, permute_dims212, out_dtype="void") split52: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul212, indices_or_sections=2, axis=-1) split_052: R.Tensor((1, seq_len, 16384), dtype="float16") = split52[0] split_152: R.Tensor((1, seq_len, 16384), dtype="float16") = split52[1] gelu52: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_052) mul55: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu52, split_152) permute_dims213: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_16_mlp_down_proj_weight3, axes=None) matmul213: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul55, permute_dims213, out_dtype="void") add105: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul213, add104) rms_norm108: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add105, model_layers_17_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims214: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_17_self_attn_qkv_proj_weight3, axes=None) matmul214: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm108, permute_dims214, out_dtype="void") reshape212: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul214, R.shape([1, seq_len, 10, 256])) reshape213: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape212, R.shape([seq_len, 10, 256])) lv54 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape213), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape214: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv54, R.shape([1, seq_len, 8, 256])) reshape215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape214, R.shape([1, seq_len, 2048])) permute_dims215: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_17_self_attn_o_proj_weight3, axes=None) matmul215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape215, permute_dims215, out_dtype="void") add106: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul215, add105) rms_norm109: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add106, model_layers_17_post_attention_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims216: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_17_mlp_gate_up_proj_weight3, axes=None) matmul216: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm109, permute_dims216, out_dtype="void") split53: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul216, indices_or_sections=2, axis=-1) split_053: R.Tensor((1, seq_len, 16384), dtype="float16") = split53[0] split_153: R.Tensor((1, seq_len, 16384), dtype="float16") = split53[1] gelu53: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_053) mul56: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu53, split_153) permute_dims217: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_17_mlp_down_proj_weight3, axes=None) matmul217: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul56, permute_dims217, out_dtype="void") add107: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul217, add106) rms_norm110: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add107, model_norm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) take1: R.Tensor((1, batch_size, 2048), dtype="float16") = R.take(rms_norm110, logit_positions, axis=1) permute_dims218: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight3, axes=None) matmul218: R.Tensor((1, batch_size, vocab_size), dtype="float32") = R.matmul(take1, permute_dims218, out_dtype="float32") gv3: R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype="float32"), R.Object) = matmul218, paged_kv_cache R.output(gv3) return gv3 @R.function def batch_verify(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) with R.dataflow(): model_embed_tokens_weight5: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[109] mul76: R.Tensor((1, seq_len, 2048), dtype="float16") = R.multiply(input_embeds, R.const(45.25, "float16")) rms_norm148: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(mul76, model_layers_0_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims292: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight5, axes=None) matmul292: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm148, permute_dims292, out_dtype="void") reshape288: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul292, R.shape([1, seq_len, 10, 256])) reshape289: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape288, R.shape([seq_len, 10, 256])) lv73 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape289), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape290: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv73, R.shape([1, seq_len, 8, 256])) reshape291: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape290, R.shape([1, seq_len, 2048])) permute_dims293: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight5, axes=None) matmul293: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape291, permute_dims293, out_dtype="void") add144: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul293, mul76) rms_norm149: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add144, model_layers_0_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims294: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight5, axes=None) matmul294: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm149, permute_dims294, out_dtype="void") split72: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul294, indices_or_sections=2, axis=-1) split_072: R.Tensor((1, seq_len, 16384), dtype="float16") = split72[0] split_172: R.Tensor((1, seq_len, 16384), dtype="float16") = split72[1] gelu72: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_072) mul77: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu72, split_172) permute_dims295: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight5, axes=None) matmul295: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul77, permute_dims295, out_dtype="void") add145: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul295, add144) rms_norm150: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add145, model_layers_1_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims296: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_1_self_attn_qkv_proj_weight5, axes=None) matmul296: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm150, permute_dims296, out_dtype="void") reshape292: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul296, R.shape([1, seq_len, 10, 256])) reshape293: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape292, R.shape([seq_len, 10, 256])) lv74 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape293), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape294: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv74, R.shape([1, seq_len, 8, 256])) reshape295: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape294, R.shape([1, seq_len, 2048])) permute_dims297: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_1_self_attn_o_proj_weight5, axes=None) matmul297: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape295, permute_dims297, out_dtype="void") add146: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul297, add145) rms_norm151: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add146, model_layers_1_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims298: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_1_mlp_gate_up_proj_weight5, axes=None) matmul298: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm151, permute_dims298, out_dtype="void") split73: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul298, indices_or_sections=2, axis=-1) split_073: R.Tensor((1, seq_len, 16384), dtype="float16") = split73[0] split_173: R.Tensor((1, seq_len, 16384), dtype="float16") = split73[1] gelu73: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_073) mul78: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu73, split_173) permute_dims299: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_1_mlp_down_proj_weight5, axes=None) matmul299: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul78, permute_dims299, out_dtype="void") add147: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul299, add146) rms_norm152: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add147, model_layers_2_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims300: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_2_self_attn_qkv_proj_weight5, axes=None) matmul300: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm152, permute_dims300, out_dtype="void") reshape296: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul300, R.shape([1, seq_len, 10, 256])) reshape297: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape296, R.shape([seq_len, 10, 256])) lv75 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape297), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape298: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv75, R.shape([1, seq_len, 8, 256])) reshape299: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape298, R.shape([1, seq_len, 2048])) permute_dims301: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_2_self_attn_o_proj_weight5, axes=None) matmul301: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape299, permute_dims301, out_dtype="void") add148: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul301, add147) rms_norm153: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add148, model_layers_2_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims302: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_2_mlp_gate_up_proj_weight5, axes=None) matmul302: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm153, permute_dims302, out_dtype="void") split74: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul302, indices_or_sections=2, axis=-1) split_074: R.Tensor((1, seq_len, 16384), dtype="float16") = split74[0] split_174: R.Tensor((1, seq_len, 16384), dtype="float16") = split74[1] gelu74: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_074) mul79: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu74, split_174) permute_dims303: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_2_mlp_down_proj_weight5, axes=None) matmul303: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul79, permute_dims303, out_dtype="void") add149: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul303, add148) rms_norm154: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add149, model_layers_3_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims304: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_3_self_attn_qkv_proj_weight5, axes=None) matmul304: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm154, permute_dims304, out_dtype="void") reshape300: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul304, R.shape([1, seq_len, 10, 256])) reshape301: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape300, R.shape([seq_len, 10, 256])) lv76 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape301), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape302: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv76, R.shape([1, seq_len, 8, 256])) reshape303: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape302, R.shape([1, seq_len, 2048])) permute_dims305: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_3_self_attn_o_proj_weight5, axes=None) matmul305: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape303, permute_dims305, out_dtype="void") add150: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul305, add149) rms_norm155: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add150, model_layers_3_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims306: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_3_mlp_gate_up_proj_weight5, axes=None) matmul306: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm155, permute_dims306, out_dtype="void") split75: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul306, indices_or_sections=2, axis=-1) split_075: R.Tensor((1, seq_len, 16384), dtype="float16") = split75[0] split_175: R.Tensor((1, seq_len, 16384), dtype="float16") = split75[1] gelu75: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_075) mul80: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu75, split_175) permute_dims307: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_3_mlp_down_proj_weight5, axes=None) matmul307: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul80, permute_dims307, out_dtype="void") add151: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul307, add150) rms_norm156: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add151, model_layers_4_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims308: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_4_self_attn_qkv_proj_weight5, axes=None) matmul308: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm156, permute_dims308, out_dtype="void") reshape304: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul308, R.shape([1, seq_len, 10, 256])) reshape305: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape304, R.shape([seq_len, 10, 256])) lv77 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape305), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape306: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv77, R.shape([1, seq_len, 8, 256])) reshape307: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape306, R.shape([1, seq_len, 2048])) permute_dims309: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_4_self_attn_o_proj_weight5, axes=None) matmul309: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape307, permute_dims309, out_dtype="void") add152: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul309, add151) rms_norm157: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add152, model_layers_4_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims310: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_4_mlp_gate_up_proj_weight5, axes=None) matmul310: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm157, permute_dims310, out_dtype="void") split76: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul310, indices_or_sections=2, axis=-1) split_076: R.Tensor((1, seq_len, 16384), dtype="float16") = split76[0] split_176: R.Tensor((1, seq_len, 16384), dtype="float16") = split76[1] gelu76: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_076) mul81: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu76, split_176) permute_dims311: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_4_mlp_down_proj_weight5, axes=None) matmul311: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul81, permute_dims311, out_dtype="void") add153: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul311, add152) rms_norm158: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add153, model_layers_5_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims312: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_5_self_attn_qkv_proj_weight5, axes=None) matmul312: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm158, permute_dims312, out_dtype="void") reshape308: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul312, R.shape([1, seq_len, 10, 256])) reshape309: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape308, R.shape([seq_len, 10, 256])) lv78 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape309), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape310: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv78, R.shape([1, seq_len, 8, 256])) reshape311: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape310, R.shape([1, seq_len, 2048])) permute_dims313: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_5_self_attn_o_proj_weight5, axes=None) matmul313: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape311, permute_dims313, out_dtype="void") add154: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul313, add153) rms_norm159: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add154, model_layers_5_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims314: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_5_mlp_gate_up_proj_weight5, axes=None) matmul314: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm159, permute_dims314, out_dtype="void") split77: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul314, indices_or_sections=2, axis=-1) split_077: R.Tensor((1, seq_len, 16384), dtype="float16") = split77[0] split_177: R.Tensor((1, seq_len, 16384), dtype="float16") = split77[1] gelu77: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_077) mul82: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu77, split_177) permute_dims315: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_5_mlp_down_proj_weight5, axes=None) matmul315: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul82, permute_dims315, out_dtype="void") add155: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul315, add154) rms_norm160: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add155, model_layers_6_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims316: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_6_self_attn_qkv_proj_weight5, axes=None) matmul316: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm160, permute_dims316, out_dtype="void") reshape312: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul316, R.shape([1, seq_len, 10, 256])) reshape313: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape312, R.shape([seq_len, 10, 256])) lv79 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape313), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape314: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv79, R.shape([1, seq_len, 8, 256])) reshape315: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape314, R.shape([1, seq_len, 2048])) permute_dims317: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_6_self_attn_o_proj_weight5, axes=None) matmul317: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape315, permute_dims317, out_dtype="void") add156: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul317, add155) rms_norm161: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add156, model_layers_6_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims318: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_6_mlp_gate_up_proj_weight5, axes=None) matmul318: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm161, permute_dims318, out_dtype="void") split78: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul318, indices_or_sections=2, axis=-1) split_078: R.Tensor((1, seq_len, 16384), dtype="float16") = split78[0] split_178: R.Tensor((1, seq_len, 16384), dtype="float16") = split78[1] gelu78: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_078) mul83: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu78, split_178) permute_dims319: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_6_mlp_down_proj_weight5, axes=None) matmul319: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul83, permute_dims319, out_dtype="void") add157: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul319, add156) rms_norm162: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add157, model_layers_7_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims320: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_7_self_attn_qkv_proj_weight5, axes=None) matmul320: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm162, permute_dims320, out_dtype="void") reshape316: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul320, R.shape([1, seq_len, 10, 256])) reshape317: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape316, R.shape([seq_len, 10, 256])) lv80 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape317), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape318: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv80, R.shape([1, seq_len, 8, 256])) reshape319: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape318, R.shape([1, seq_len, 2048])) permute_dims321: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_7_self_attn_o_proj_weight5, axes=None) matmul321: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape319, permute_dims321, out_dtype="void") add158: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul321, add157) rms_norm163: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add158, model_layers_7_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims322: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_7_mlp_gate_up_proj_weight5, axes=None) matmul322: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm163, permute_dims322, out_dtype="void") split79: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul322, indices_or_sections=2, axis=-1) split_079: R.Tensor((1, seq_len, 16384), dtype="float16") = split79[0] split_179: R.Tensor((1, seq_len, 16384), dtype="float16") = split79[1] gelu79: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_079) mul84: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu79, split_179) permute_dims323: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_7_mlp_down_proj_weight5, axes=None) matmul323: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul84, permute_dims323, out_dtype="void") add159: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul323, add158) rms_norm164: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add159, model_layers_8_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims324: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_8_self_attn_qkv_proj_weight5, axes=None) matmul324: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm164, permute_dims324, out_dtype="void") reshape320: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul324, R.shape([1, seq_len, 10, 256])) reshape321: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape320, R.shape([seq_len, 10, 256])) lv81 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape321), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape322: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv81, R.shape([1, seq_len, 8, 256])) reshape323: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape322, R.shape([1, seq_len, 2048])) permute_dims325: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_8_self_attn_o_proj_weight5, axes=None) matmul325: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape323, permute_dims325, out_dtype="void") add160: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul325, add159) rms_norm165: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add160, model_layers_8_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims326: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_8_mlp_gate_up_proj_weight5, axes=None) matmul326: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm165, permute_dims326, out_dtype="void") split80: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul326, indices_or_sections=2, axis=-1) split_080: R.Tensor((1, seq_len, 16384), dtype="float16") = split80[0] split_180: R.Tensor((1, seq_len, 16384), dtype="float16") = split80[1] gelu80: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_080) mul85: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu80, split_180) permute_dims327: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_8_mlp_down_proj_weight5, axes=None) matmul327: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul85, permute_dims327, out_dtype="void") add161: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul327, add160) rms_norm166: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add161, model_layers_9_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims328: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_9_self_attn_qkv_proj_weight5, axes=None) matmul328: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm166, permute_dims328, out_dtype="void") reshape324: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul328, R.shape([1, seq_len, 10, 256])) reshape325: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape324, R.shape([seq_len, 10, 256])) lv82 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape325), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape326: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv82, R.shape([1, seq_len, 8, 256])) reshape327: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape326, R.shape([1, seq_len, 2048])) permute_dims329: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_9_self_attn_o_proj_weight5, axes=None) matmul329: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape327, permute_dims329, out_dtype="void") add162: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul329, add161) rms_norm167: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add162, model_layers_9_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims330: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_9_mlp_gate_up_proj_weight5, axes=None) matmul330: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm167, permute_dims330, out_dtype="void") split81: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul330, indices_or_sections=2, axis=-1) split_081: R.Tensor((1, seq_len, 16384), dtype="float16") = split81[0] split_181: R.Tensor((1, seq_len, 16384), dtype="float16") = split81[1] gelu81: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_081) mul86: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu81, split_181) permute_dims331: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_9_mlp_down_proj_weight5, axes=None) matmul331: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul86, permute_dims331, out_dtype="void") add163: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul331, add162) rms_norm168: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add163, model_layers_10_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims332: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_10_self_attn_qkv_proj_weight5, axes=None) matmul332: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm168, permute_dims332, out_dtype="void") reshape328: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul332, R.shape([1, seq_len, 10, 256])) reshape329: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape328, R.shape([seq_len, 10, 256])) lv83 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape329), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape330: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv83, R.shape([1, seq_len, 8, 256])) reshape331: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape330, R.shape([1, seq_len, 2048])) permute_dims333: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_10_self_attn_o_proj_weight5, axes=None) matmul333: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape331, permute_dims333, out_dtype="void") add164: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul333, add163) rms_norm169: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add164, model_layers_10_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims334: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_10_mlp_gate_up_proj_weight5, axes=None) matmul334: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm169, permute_dims334, out_dtype="void") split82: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul334, indices_or_sections=2, axis=-1) split_082: R.Tensor((1, seq_len, 16384), dtype="float16") = split82[0] split_182: R.Tensor((1, seq_len, 16384), dtype="float16") = split82[1] gelu82: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_082) mul87: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu82, split_182) permute_dims335: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_10_mlp_down_proj_weight5, axes=None) matmul335: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul87, permute_dims335, out_dtype="void") add165: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul335, add164) rms_norm170: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add165, model_layers_11_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims336: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_11_self_attn_qkv_proj_weight5, axes=None) matmul336: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm170, permute_dims336, out_dtype="void") reshape332: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul336, R.shape([1, seq_len, 10, 256])) reshape333: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape332, R.shape([seq_len, 10, 256])) lv84 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape333), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape334: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv84, R.shape([1, seq_len, 8, 256])) reshape335: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape334, R.shape([1, seq_len, 2048])) permute_dims337: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_11_self_attn_o_proj_weight5, axes=None) matmul337: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape335, permute_dims337, out_dtype="void") add166: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul337, add165) rms_norm171: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add166, model_layers_11_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims338: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_11_mlp_gate_up_proj_weight5, axes=None) matmul338: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm171, permute_dims338, out_dtype="void") split83: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul338, indices_or_sections=2, axis=-1) split_083: R.Tensor((1, seq_len, 16384), dtype="float16") = split83[0] split_183: R.Tensor((1, seq_len, 16384), dtype="float16") = split83[1] gelu83: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_083) mul88: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu83, split_183) permute_dims339: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_11_mlp_down_proj_weight5, axes=None) matmul339: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul88, permute_dims339, out_dtype="void") add167: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul339, add166) rms_norm172: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add167, model_layers_12_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims340: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_12_self_attn_qkv_proj_weight5, axes=None) matmul340: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm172, permute_dims340, out_dtype="void") reshape336: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul340, R.shape([1, seq_len, 10, 256])) reshape337: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape336, R.shape([seq_len, 10, 256])) lv85 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape337), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape338: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv85, R.shape([1, seq_len, 8, 256])) reshape339: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape338, R.shape([1, seq_len, 2048])) permute_dims341: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_12_self_attn_o_proj_weight5, axes=None) matmul341: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape339, permute_dims341, out_dtype="void") add168: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul341, add167) rms_norm173: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add168, model_layers_12_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims342: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_12_mlp_gate_up_proj_weight5, axes=None) matmul342: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm173, permute_dims342, out_dtype="void") split84: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul342, indices_or_sections=2, axis=-1) split_084: R.Tensor((1, seq_len, 16384), dtype="float16") = split84[0] split_184: R.Tensor((1, seq_len, 16384), dtype="float16") = split84[1] gelu84: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_084) mul89: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu84, split_184) permute_dims343: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_12_mlp_down_proj_weight5, axes=None) matmul343: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul89, permute_dims343, out_dtype="void") add169: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul343, add168) rms_norm174: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add169, model_layers_13_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims344: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_13_self_attn_qkv_proj_weight5, axes=None) matmul344: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm174, permute_dims344, out_dtype="void") reshape340: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul344, R.shape([1, seq_len, 10, 256])) reshape341: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape340, R.shape([seq_len, 10, 256])) lv86 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape341), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape342: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv86, R.shape([1, seq_len, 8, 256])) reshape343: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape342, R.shape([1, seq_len, 2048])) permute_dims345: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_13_self_attn_o_proj_weight5, axes=None) matmul345: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape343, permute_dims345, out_dtype="void") add170: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul345, add169) rms_norm175: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add170, model_layers_13_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims346: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_13_mlp_gate_up_proj_weight5, axes=None) matmul346: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm175, permute_dims346, out_dtype="void") split85: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul346, indices_or_sections=2, axis=-1) split_085: R.Tensor((1, seq_len, 16384), dtype="float16") = split85[0] split_185: R.Tensor((1, seq_len, 16384), dtype="float16") = split85[1] gelu85: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_085) mul90: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu85, split_185) permute_dims347: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_13_mlp_down_proj_weight5, axes=None) matmul347: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul90, permute_dims347, out_dtype="void") add171: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul347, add170) rms_norm176: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add171, model_layers_14_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims348: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_14_self_attn_qkv_proj_weight5, axes=None) matmul348: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm176, permute_dims348, out_dtype="void") reshape344: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul348, R.shape([1, seq_len, 10, 256])) reshape345: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape344, R.shape([seq_len, 10, 256])) lv87 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape345), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape346: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv87, R.shape([1, seq_len, 8, 256])) reshape347: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape346, R.shape([1, seq_len, 2048])) permute_dims349: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_14_self_attn_o_proj_weight5, axes=None) matmul349: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape347, permute_dims349, out_dtype="void") add172: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul349, add171) rms_norm177: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add172, model_layers_14_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims350: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_14_mlp_gate_up_proj_weight5, axes=None) matmul350: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm177, permute_dims350, out_dtype="void") split86: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul350, indices_or_sections=2, axis=-1) split_086: R.Tensor((1, seq_len, 16384), dtype="float16") = split86[0] split_186: R.Tensor((1, seq_len, 16384), dtype="float16") = split86[1] gelu86: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_086) mul91: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu86, split_186) permute_dims351: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_14_mlp_down_proj_weight5, axes=None) matmul351: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul91, permute_dims351, out_dtype="void") add173: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul351, add172) rms_norm178: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add173, model_layers_15_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims352: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_15_self_attn_qkv_proj_weight5, axes=None) matmul352: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm178, permute_dims352, out_dtype="void") reshape348: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul352, R.shape([1, seq_len, 10, 256])) reshape349: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape348, R.shape([seq_len, 10, 256])) lv88 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape349), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape350: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv88, R.shape([1, seq_len, 8, 256])) reshape351: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape350, R.shape([1, seq_len, 2048])) permute_dims353: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_15_self_attn_o_proj_weight5, axes=None) matmul353: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape351, permute_dims353, out_dtype="void") add174: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul353, add173) rms_norm179: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add174, model_layers_15_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims354: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_15_mlp_gate_up_proj_weight5, axes=None) matmul354: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm179, permute_dims354, out_dtype="void") split87: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul354, indices_or_sections=2, axis=-1) split_087: R.Tensor((1, seq_len, 16384), dtype="float16") = split87[0] split_187: R.Tensor((1, seq_len, 16384), dtype="float16") = split87[1] gelu87: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_087) mul92: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu87, split_187) permute_dims355: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_15_mlp_down_proj_weight5, axes=None) matmul355: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul92, permute_dims355, out_dtype="void") add175: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul355, add174) rms_norm180: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add175, model_layers_16_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims356: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_16_self_attn_qkv_proj_weight5, axes=None) matmul356: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm180, permute_dims356, out_dtype="void") reshape352: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul356, R.shape([1, seq_len, 10, 256])) reshape353: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape352, R.shape([seq_len, 10, 256])) lv89 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape353), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape354: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv89, R.shape([1, seq_len, 8, 256])) reshape355: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape354, R.shape([1, seq_len, 2048])) permute_dims357: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_16_self_attn_o_proj_weight5, axes=None) matmul357: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape355, permute_dims357, out_dtype="void") add176: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul357, add175) rms_norm181: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add176, model_layers_16_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims358: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_16_mlp_gate_up_proj_weight5, axes=None) matmul358: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm181, permute_dims358, out_dtype="void") split88: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul358, indices_or_sections=2, axis=-1) split_088: R.Tensor((1, seq_len, 16384), dtype="float16") = split88[0] split_188: R.Tensor((1, seq_len, 16384), dtype="float16") = split88[1] gelu88: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_088) mul93: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu88, split_188) permute_dims359: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_16_mlp_down_proj_weight5, axes=None) matmul359: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul93, permute_dims359, out_dtype="void") add177: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul359, add176) rms_norm182: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add177, model_layers_17_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims360: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_17_self_attn_qkv_proj_weight5, axes=None) matmul360: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm182, permute_dims360, out_dtype="void") reshape356: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul360, R.shape([1, seq_len, 10, 256])) reshape357: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape356, R.shape([seq_len, 10, 256])) lv90 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape357), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape358: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv90, R.shape([1, seq_len, 8, 256])) reshape359: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape358, R.shape([1, seq_len, 2048])) permute_dims361: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_17_self_attn_o_proj_weight5, axes=None) matmul361: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape359, permute_dims361, out_dtype="void") add178: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul361, add177) rms_norm183: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add178, model_layers_17_post_attention_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims362: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_17_mlp_gate_up_proj_weight5, axes=None) matmul362: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm183, permute_dims362, out_dtype="void") split89: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul362, indices_or_sections=2, axis=-1) split_089: R.Tensor((1, seq_len, 16384), dtype="float16") = split89[0] split_189: R.Tensor((1, seq_len, 16384), dtype="float16") = split89[1] gelu89: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_089) mul94: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu89, split_189) permute_dims363: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_17_mlp_down_proj_weight5, axes=None) matmul363: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul94, permute_dims363, out_dtype="void") add179: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul363, add178) rms_norm184: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add179, model_norm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims364: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight5, axes=None) matmul364: R.Tensor((1, seq_len, vocab_size), dtype="float32") = R.matmul(rms_norm184, permute_dims364, out_dtype="float32") gv5: R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype="float32"), R.Object) = matmul364, paged_kv_cache R.output(gv5) return gv5 @R.function def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: max_batch_size = T.int64() max_total_seq_len = T.int64() prefill_chunk_size = T.int64() page_size = T.int64() support_sliding_window = T.int64() R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module gv: R.Tensor((), dtype="float16") = R.zeros(R.shape([]), dtype="float16") paged_kv_cache: R.Object = R.call_pure_packed("vm.builtin.paged_attention_kv_cache_create_reduced", R.shape([max_batch_size, max_total_seq_len, prefill_chunk_size, page_size, support_sliding_window]), R.shape([0, 18]), R.prim_value(8), R.prim_value(1), R.prim_value(256), R.prim_value(1), R.prim_value(1), R.prim_value(T.float32(10000.0)), gv, cls.tir_kv_cache_transpose_append, cls.batch_prefill_paged_kv, cls.batch_decode_paged_kv, cls.batch_prefill_paged_kv_sliding_window, cls.batch_decode_paged_kv_sliding_window, cls.batch_prefill_ragged_kv, cls.merge_state_inplace, cls.fused_rope, cls.copy_single_page, cls.tir_kv_cache_debug_get_kv, cls.compact_kv_copy, cls.batch_tree_attn, cls.tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) return paged_kv_cache @R.function def decode(input_embed: R.Tensor((1, 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) with R.dataflow(): model_embed_tokens_weight2: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[109] mul19: R.Tensor((1, 1, 2048), dtype="float16") = R.multiply(input_embed, R.const(45.25, "float16")) rms_norm37: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(mul19, model_layers_0_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims73: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight2, axes=None) matmul73: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm37, permute_dims73, out_dtype="void") reshape72: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul73, R.shape([1, 1, 10, 256])) reshape73: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape72, R.shape([1, 10, 256])) lv19 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape73), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape74: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv19, R.shape([1, 1, 8, 256])) reshape75: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape74, R.shape([1, 1, 2048])) permute_dims74: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight2, axes=None) matmul74: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape75, permute_dims74, out_dtype="void") add36: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul74, mul19) rms_norm38: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add36, model_layers_0_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims75: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight2, axes=None) matmul75: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm38, permute_dims75, out_dtype="void") split18: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul75, indices_or_sections=2, axis=-1) split_018: R.Tensor((1, 1, 16384), dtype="float16") = split18[0] split_118: R.Tensor((1, 1, 16384), dtype="float16") = split18[1] gelu18: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_018) mul20: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu18, split_118) permute_dims76: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight2, axes=None) matmul76: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul20, permute_dims76, out_dtype="void") add37: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul76, add36) rms_norm39: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add37, model_layers_1_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims77: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_1_self_attn_qkv_proj_weight2, axes=None) matmul77: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm39, permute_dims77, out_dtype="void") reshape76: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul77, R.shape([1, 1, 10, 256])) reshape77: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape76, R.shape([1, 10, 256])) lv20 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape77), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape78: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv20, R.shape([1, 1, 8, 256])) reshape79: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape78, R.shape([1, 1, 2048])) permute_dims78: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_1_self_attn_o_proj_weight2, axes=None) matmul78: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape79, permute_dims78, out_dtype="void") add38: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul78, add37) rms_norm40: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add38, model_layers_1_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims79: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_1_mlp_gate_up_proj_weight2, axes=None) matmul79: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm40, permute_dims79, out_dtype="void") split19: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul79, indices_or_sections=2, axis=-1) split_019: R.Tensor((1, 1, 16384), dtype="float16") = split19[0] split_119: R.Tensor((1, 1, 16384), dtype="float16") = split19[1] gelu19: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_019) mul21: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu19, split_119) permute_dims80: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_1_mlp_down_proj_weight2, axes=None) matmul80: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul21, permute_dims80, out_dtype="void") add39: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul80, add38) rms_norm41: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add39, model_layers_2_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims81: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_2_self_attn_qkv_proj_weight2, axes=None) matmul81: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm41, permute_dims81, out_dtype="void") reshape80: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul81, R.shape([1, 1, 10, 256])) reshape81: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape80, R.shape([1, 10, 256])) lv21 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape81), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape82: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv21, R.shape([1, 1, 8, 256])) reshape83: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape82, R.shape([1, 1, 2048])) permute_dims82: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_2_self_attn_o_proj_weight2, axes=None) matmul82: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape83, permute_dims82, out_dtype="void") add40: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul82, add39) rms_norm42: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add40, model_layers_2_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims83: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_2_mlp_gate_up_proj_weight2, axes=None) matmul83: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm42, permute_dims83, out_dtype="void") split20: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul83, indices_or_sections=2, axis=-1) split_020: R.Tensor((1, 1, 16384), dtype="float16") = split20[0] split_120: R.Tensor((1, 1, 16384), dtype="float16") = split20[1] gelu20: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_020) mul22: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu20, split_120) permute_dims84: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_2_mlp_down_proj_weight2, axes=None) matmul84: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul22, permute_dims84, out_dtype="void") add41: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul84, add40) rms_norm43: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add41, model_layers_3_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims85: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_3_self_attn_qkv_proj_weight2, axes=None) matmul85: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm43, permute_dims85, out_dtype="void") reshape84: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul85, R.shape([1, 1, 10, 256])) reshape85: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape84, R.shape([1, 10, 256])) lv22 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape85), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape86: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv22, R.shape([1, 1, 8, 256])) reshape87: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape86, R.shape([1, 1, 2048])) permute_dims86: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_3_self_attn_o_proj_weight2, axes=None) matmul86: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape87, permute_dims86, out_dtype="void") add42: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul86, add41) rms_norm44: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add42, model_layers_3_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims87: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_3_mlp_gate_up_proj_weight2, axes=None) matmul87: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm44, permute_dims87, out_dtype="void") split21: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul87, indices_or_sections=2, axis=-1) split_021: R.Tensor((1, 1, 16384), dtype="float16") = split21[0] split_121: R.Tensor((1, 1, 16384), dtype="float16") = split21[1] gelu21: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_021) mul23: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu21, split_121) permute_dims88: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_3_mlp_down_proj_weight2, axes=None) matmul88: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul23, permute_dims88, out_dtype="void") add43: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul88, add42) rms_norm45: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add43, model_layers_4_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims89: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_4_self_attn_qkv_proj_weight2, axes=None) matmul89: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm45, permute_dims89, out_dtype="void") reshape88: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul89, R.shape([1, 1, 10, 256])) reshape89: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape88, R.shape([1, 10, 256])) lv23 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape89), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape90: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv23, R.shape([1, 1, 8, 256])) reshape91: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape90, R.shape([1, 1, 2048])) permute_dims90: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_4_self_attn_o_proj_weight2, axes=None) matmul90: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape91, permute_dims90, out_dtype="void") add44: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul90, add43) rms_norm46: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add44, model_layers_4_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims91: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_4_mlp_gate_up_proj_weight2, axes=None) matmul91: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm46, permute_dims91, out_dtype="void") split22: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul91, indices_or_sections=2, axis=-1) split_022: R.Tensor((1, 1, 16384), dtype="float16") = split22[0] split_122: R.Tensor((1, 1, 16384), dtype="float16") = split22[1] gelu22: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_022) mul24: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu22, split_122) permute_dims92: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_4_mlp_down_proj_weight2, axes=None) matmul92: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul24, permute_dims92, out_dtype="void") add45: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul92, add44) rms_norm47: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add45, model_layers_5_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims93: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_5_self_attn_qkv_proj_weight2, axes=None) matmul93: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm47, permute_dims93, out_dtype="void") reshape92: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul93, R.shape([1, 1, 10, 256])) reshape93: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape92, R.shape([1, 10, 256])) lv24 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape93), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape94: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv24, R.shape([1, 1, 8, 256])) reshape95: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape94, R.shape([1, 1, 2048])) permute_dims94: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_5_self_attn_o_proj_weight2, axes=None) matmul94: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape95, permute_dims94, out_dtype="void") add46: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul94, add45) rms_norm48: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add46, model_layers_5_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims95: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_5_mlp_gate_up_proj_weight2, axes=None) matmul95: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm48, permute_dims95, out_dtype="void") split23: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul95, indices_or_sections=2, axis=-1) split_023: R.Tensor((1, 1, 16384), dtype="float16") = split23[0] split_123: R.Tensor((1, 1, 16384), dtype="float16") = split23[1] gelu23: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_023) mul25: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu23, split_123) permute_dims96: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_5_mlp_down_proj_weight2, axes=None) matmul96: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul25, permute_dims96, out_dtype="void") add47: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul96, add46) rms_norm49: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add47, model_layers_6_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims97: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_6_self_attn_qkv_proj_weight2, axes=None) matmul97: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm49, permute_dims97, out_dtype="void") reshape96: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul97, R.shape([1, 1, 10, 256])) reshape97: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape96, R.shape([1, 10, 256])) lv25 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape97), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape98: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv25, R.shape([1, 1, 8, 256])) reshape99: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape98, R.shape([1, 1, 2048])) permute_dims98: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_6_self_attn_o_proj_weight2, axes=None) matmul98: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape99, permute_dims98, out_dtype="void") add48: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul98, add47) rms_norm50: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add48, model_layers_6_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims99: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_6_mlp_gate_up_proj_weight2, axes=None) matmul99: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm50, permute_dims99, out_dtype="void") split24: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul99, indices_or_sections=2, axis=-1) split_024: R.Tensor((1, 1, 16384), dtype="float16") = split24[0] split_124: R.Tensor((1, 1, 16384), dtype="float16") = split24[1] gelu24: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_024) mul26: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu24, split_124) permute_dims100: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_6_mlp_down_proj_weight2, axes=None) matmul100: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul26, permute_dims100, out_dtype="void") add49: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul100, add48) rms_norm51: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add49, model_layers_7_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims101: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_7_self_attn_qkv_proj_weight2, axes=None) matmul101: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm51, permute_dims101, out_dtype="void") reshape100: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul101, R.shape([1, 1, 10, 256])) reshape101: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape100, R.shape([1, 10, 256])) lv26 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape101), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape102: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv26, R.shape([1, 1, 8, 256])) reshape103: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape102, R.shape([1, 1, 2048])) permute_dims102: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_7_self_attn_o_proj_weight2, axes=None) matmul102: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape103, permute_dims102, out_dtype="void") add50: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul102, add49) rms_norm52: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add50, model_layers_7_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims103: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_7_mlp_gate_up_proj_weight2, axes=None) matmul103: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm52, permute_dims103, out_dtype="void") split25: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul103, indices_or_sections=2, axis=-1) split_025: R.Tensor((1, 1, 16384), dtype="float16") = split25[0] split_125: R.Tensor((1, 1, 16384), dtype="float16") = split25[1] gelu25: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_025) mul27: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu25, split_125) permute_dims104: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_7_mlp_down_proj_weight2, axes=None) matmul104: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul27, permute_dims104, out_dtype="void") add51: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul104, add50) rms_norm53: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add51, model_layers_8_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims105: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_8_self_attn_qkv_proj_weight2, axes=None) matmul105: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm53, permute_dims105, out_dtype="void") reshape104: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul105, R.shape([1, 1, 10, 256])) reshape105: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape104, R.shape([1, 10, 256])) lv27 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape105), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape106: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv27, R.shape([1, 1, 8, 256])) reshape107: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape106, R.shape([1, 1, 2048])) permute_dims106: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_8_self_attn_o_proj_weight2, axes=None) matmul106: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape107, permute_dims106, out_dtype="void") add52: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul106, add51) rms_norm54: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add52, model_layers_8_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims107: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_8_mlp_gate_up_proj_weight2, axes=None) matmul107: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm54, permute_dims107, out_dtype="void") split26: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul107, indices_or_sections=2, axis=-1) split_026: R.Tensor((1, 1, 16384), dtype="float16") = split26[0] split_126: R.Tensor((1, 1, 16384), dtype="float16") = split26[1] gelu26: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_026) mul28: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu26, split_126) permute_dims108: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_8_mlp_down_proj_weight2, axes=None) matmul108: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul28, permute_dims108, out_dtype="void") add53: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul108, add52) rms_norm55: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add53, model_layers_9_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims109: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_9_self_attn_qkv_proj_weight2, axes=None) matmul109: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm55, permute_dims109, out_dtype="void") reshape108: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul109, R.shape([1, 1, 10, 256])) reshape109: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape108, R.shape([1, 10, 256])) lv28 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape109), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape110: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv28, R.shape([1, 1, 8, 256])) reshape111: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape110, R.shape([1, 1, 2048])) permute_dims110: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_9_self_attn_o_proj_weight2, axes=None) matmul110: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape111, permute_dims110, out_dtype="void") add54: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul110, add53) rms_norm56: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add54, model_layers_9_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims111: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_9_mlp_gate_up_proj_weight2, axes=None) matmul111: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm56, permute_dims111, out_dtype="void") split27: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul111, indices_or_sections=2, axis=-1) split_027: R.Tensor((1, 1, 16384), dtype="float16") = split27[0] split_127: R.Tensor((1, 1, 16384), dtype="float16") = split27[1] gelu27: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_027) mul29: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu27, split_127) permute_dims112: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_9_mlp_down_proj_weight2, axes=None) matmul112: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul29, permute_dims112, out_dtype="void") add55: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul112, add54) rms_norm57: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add55, model_layers_10_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims113: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_10_self_attn_qkv_proj_weight2, axes=None) matmul113: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm57, permute_dims113, out_dtype="void") reshape112: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul113, R.shape([1, 1, 10, 256])) reshape113: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape112, R.shape([1, 10, 256])) lv29 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape113), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape114: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv29, R.shape([1, 1, 8, 256])) reshape115: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape114, R.shape([1, 1, 2048])) permute_dims114: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_10_self_attn_o_proj_weight2, axes=None) matmul114: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape115, permute_dims114, out_dtype="void") add56: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul114, add55) rms_norm58: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add56, model_layers_10_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims115: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_10_mlp_gate_up_proj_weight2, axes=None) matmul115: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm58, permute_dims115, out_dtype="void") split28: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul115, indices_or_sections=2, axis=-1) split_028: R.Tensor((1, 1, 16384), dtype="float16") = split28[0] split_128: R.Tensor((1, 1, 16384), dtype="float16") = split28[1] gelu28: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_028) mul30: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu28, split_128) permute_dims116: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_10_mlp_down_proj_weight2, axes=None) matmul116: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul30, permute_dims116, out_dtype="void") add57: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul116, add56) rms_norm59: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add57, model_layers_11_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims117: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_11_self_attn_qkv_proj_weight2, axes=None) matmul117: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm59, permute_dims117, out_dtype="void") reshape116: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul117, R.shape([1, 1, 10, 256])) reshape117: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape116, R.shape([1, 10, 256])) lv30 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape117), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape118: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv30, R.shape([1, 1, 8, 256])) reshape119: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape118, R.shape([1, 1, 2048])) permute_dims118: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_11_self_attn_o_proj_weight2, axes=None) matmul118: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape119, permute_dims118, out_dtype="void") add58: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul118, add57) rms_norm60: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add58, model_layers_11_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims119: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_11_mlp_gate_up_proj_weight2, axes=None) matmul119: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm60, permute_dims119, out_dtype="void") split29: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul119, indices_or_sections=2, axis=-1) split_029: R.Tensor((1, 1, 16384), dtype="float16") = split29[0] split_129: R.Tensor((1, 1, 16384), dtype="float16") = split29[1] gelu29: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_029) mul31: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu29, split_129) permute_dims120: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_11_mlp_down_proj_weight2, axes=None) matmul120: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul31, permute_dims120, out_dtype="void") add59: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul120, add58) rms_norm61: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add59, model_layers_12_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims121: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_12_self_attn_qkv_proj_weight2, axes=None) matmul121: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm61, permute_dims121, out_dtype="void") reshape120: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul121, R.shape([1, 1, 10, 256])) reshape121: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape120, R.shape([1, 10, 256])) lv31 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape121), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape122: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv31, R.shape([1, 1, 8, 256])) reshape123: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape122, R.shape([1, 1, 2048])) permute_dims122: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_12_self_attn_o_proj_weight2, axes=None) matmul122: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape123, permute_dims122, out_dtype="void") add60: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul122, add59) rms_norm62: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add60, model_layers_12_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims123: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_12_mlp_gate_up_proj_weight2, axes=None) matmul123: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm62, permute_dims123, out_dtype="void") split30: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul123, indices_or_sections=2, axis=-1) split_030: R.Tensor((1, 1, 16384), dtype="float16") = split30[0] split_130: R.Tensor((1, 1, 16384), dtype="float16") = split30[1] gelu30: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_030) mul32: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu30, split_130) permute_dims124: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_12_mlp_down_proj_weight2, axes=None) matmul124: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul32, permute_dims124, out_dtype="void") add61: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul124, add60) rms_norm63: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add61, model_layers_13_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims125: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_13_self_attn_qkv_proj_weight2, axes=None) matmul125: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm63, permute_dims125, out_dtype="void") reshape124: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul125, R.shape([1, 1, 10, 256])) reshape125: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape124, R.shape([1, 10, 256])) lv32 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape125), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape126: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv32, R.shape([1, 1, 8, 256])) reshape127: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape126, R.shape([1, 1, 2048])) permute_dims126: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_13_self_attn_o_proj_weight2, axes=None) matmul126: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape127, permute_dims126, out_dtype="void") add62: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul126, add61) rms_norm64: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add62, model_layers_13_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims127: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_13_mlp_gate_up_proj_weight2, axes=None) matmul127: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm64, permute_dims127, out_dtype="void") split31: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul127, indices_or_sections=2, axis=-1) split_031: R.Tensor((1, 1, 16384), dtype="float16") = split31[0] split_131: R.Tensor((1, 1, 16384), dtype="float16") = split31[1] gelu31: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_031) mul33: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu31, split_131) permute_dims128: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_13_mlp_down_proj_weight2, axes=None) matmul128: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul33, permute_dims128, out_dtype="void") add63: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul128, add62) rms_norm65: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add63, model_layers_14_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims129: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_14_self_attn_qkv_proj_weight2, axes=None) matmul129: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm65, permute_dims129, out_dtype="void") reshape128: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul129, R.shape([1, 1, 10, 256])) reshape129: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape128, R.shape([1, 10, 256])) lv33 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape129), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape130: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv33, R.shape([1, 1, 8, 256])) reshape131: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape130, R.shape([1, 1, 2048])) permute_dims130: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_14_self_attn_o_proj_weight2, axes=None) matmul130: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape131, permute_dims130, out_dtype="void") add64: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul130, add63) rms_norm66: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add64, model_layers_14_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims131: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_14_mlp_gate_up_proj_weight2, axes=None) matmul131: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm66, permute_dims131, out_dtype="void") split32: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul131, indices_or_sections=2, axis=-1) split_032: R.Tensor((1, 1, 16384), dtype="float16") = split32[0] split_132: R.Tensor((1, 1, 16384), dtype="float16") = split32[1] gelu32: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_032) mul34: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu32, split_132) permute_dims132: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_14_mlp_down_proj_weight2, axes=None) matmul132: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul34, permute_dims132, out_dtype="void") add65: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul132, add64) rms_norm67: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add65, model_layers_15_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims133: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_15_self_attn_qkv_proj_weight2, axes=None) matmul133: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm67, permute_dims133, out_dtype="void") reshape132: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul133, R.shape([1, 1, 10, 256])) reshape133: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape132, R.shape([1, 10, 256])) lv34 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape133), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape134: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv34, R.shape([1, 1, 8, 256])) reshape135: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape134, R.shape([1, 1, 2048])) permute_dims134: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_15_self_attn_o_proj_weight2, axes=None) matmul134: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape135, permute_dims134, out_dtype="void") add66: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul134, add65) rms_norm68: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add66, model_layers_15_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims135: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_15_mlp_gate_up_proj_weight2, axes=None) matmul135: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm68, permute_dims135, out_dtype="void") split33: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul135, indices_or_sections=2, axis=-1) split_033: R.Tensor((1, 1, 16384), dtype="float16") = split33[0] split_133: R.Tensor((1, 1, 16384), dtype="float16") = split33[1] gelu33: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_033) mul35: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu33, split_133) permute_dims136: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_15_mlp_down_proj_weight2, axes=None) matmul136: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul35, permute_dims136, out_dtype="void") add67: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul136, add66) rms_norm69: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add67, model_layers_16_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims137: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_16_self_attn_qkv_proj_weight2, axes=None) matmul137: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm69, permute_dims137, out_dtype="void") reshape136: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul137, R.shape([1, 1, 10, 256])) reshape137: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape136, R.shape([1, 10, 256])) lv35 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape137), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape138: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv35, R.shape([1, 1, 8, 256])) reshape139: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape138, R.shape([1, 1, 2048])) permute_dims138: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_16_self_attn_o_proj_weight2, axes=None) matmul138: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape139, permute_dims138, out_dtype="void") add68: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul138, add67) rms_norm70: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add68, model_layers_16_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims139: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_16_mlp_gate_up_proj_weight2, axes=None) matmul139: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm70, permute_dims139, out_dtype="void") split34: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul139, indices_or_sections=2, axis=-1) split_034: R.Tensor((1, 1, 16384), dtype="float16") = split34[0] split_134: R.Tensor((1, 1, 16384), dtype="float16") = split34[1] gelu34: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_034) mul36: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu34, split_134) permute_dims140: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_16_mlp_down_proj_weight2, axes=None) matmul140: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul36, permute_dims140, out_dtype="void") add69: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul140, add68) rms_norm71: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add69, model_layers_17_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims141: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_17_self_attn_qkv_proj_weight2, axes=None) matmul141: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm71, permute_dims141, out_dtype="void") reshape140: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(matmul141, R.shape([1, 1, 10, 256])) reshape141: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape140, R.shape([1, 10, 256])) lv36 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape141), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape142: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv36, R.shape([1, 1, 8, 256])) reshape143: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape142, R.shape([1, 1, 2048])) permute_dims142: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_17_self_attn_o_proj_weight2, axes=None) matmul142: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape143, permute_dims142, out_dtype="void") add70: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul142, add69) rms_norm72: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add70, model_layers_17_post_attention_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims143: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_17_mlp_gate_up_proj_weight2, axes=None) matmul143: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm72, permute_dims143, out_dtype="void") split35: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(matmul143, indices_or_sections=2, axis=-1) split_035: R.Tensor((1, 1, 16384), dtype="float16") = split35[0] split_135: R.Tensor((1, 1, 16384), dtype="float16") = split35[1] gelu35: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_035) mul37: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu35, split_135) permute_dims144: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_17_mlp_down_proj_weight2, axes=None) matmul144: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul37, permute_dims144, out_dtype="void") add71: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul144, add70) rms_norm73: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(add71, model_norm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims145: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight2, axes=None) matmul145: R.Tensor((1, 1, vocab_size), dtype="float32") = R.matmul(rms_norm73, permute_dims145, out_dtype="float32") gv2: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = matmul145, paged_kv_cache R.output(gv2) return gv2 @R.function def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tensor(("seq_len", 2048), dtype="float16"): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) with R.dataflow(): model_embed_tokens_weight: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight: R.Tensor((2048,), dtype="float16") = packed_params[109] take: R.Tensor((seq_len, 2048), dtype="float16") = R.take(model_embed_tokens_weight, input_ids, axis=0) gv: R.Tensor((seq_len, 2048), dtype="float16") = take R.output(gv) return gv @R.function def prefill(input_embed: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): model_embed_tokens_weight1: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[109] mul: R.Tensor((1, seq_len, 2048), dtype="float16") = R.multiply(input_embed, R.const(45.25, "float16")) rms_norm: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(mul, model_layers_0_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight1, axes=None) matmul: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm, permute_dims, out_dtype="void") reshape: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul, R.shape([1, seq_len, 10, 256])) reshape1: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape, R.shape([seq_len, 10, 256])) lv = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape2: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv, R.shape([1, seq_len, 8, 256])) reshape3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape2, R.shape([1, seq_len, 2048])) permute_dims1: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight1, axes=None) matmul1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape3, permute_dims1, out_dtype="void") add: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul1, mul) rms_norm1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add, model_layers_0_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims2: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight1, axes=None) matmul2: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm1, permute_dims2, out_dtype="void") split: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul2, indices_or_sections=2, axis=-1) split_0: R.Tensor((1, seq_len, 16384), dtype="float16") = split[0] split_1: R.Tensor((1, seq_len, 16384), dtype="float16") = split[1] gelu: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_0) mul1: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu, split_1) permute_dims3: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight1, axes=None) matmul3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul1, permute_dims3, out_dtype="void") add1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul3, add) rms_norm2: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add1, model_layers_1_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims4: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_1_self_attn_qkv_proj_weight1, axes=None) matmul4: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm2, permute_dims4, out_dtype="void") reshape4: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul4, R.shape([1, seq_len, 10, 256])) reshape5: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape4, R.shape([seq_len, 10, 256])) lv1 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape6: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv1, R.shape([1, seq_len, 8, 256])) reshape7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape6, R.shape([1, seq_len, 2048])) permute_dims5: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_1_self_attn_o_proj_weight1, axes=None) matmul5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape7, permute_dims5, out_dtype="void") add2: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul5, add1) rms_norm3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add2, model_layers_1_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims6: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_1_mlp_gate_up_proj_weight1, axes=None) matmul6: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm3, permute_dims6, out_dtype="void") split1: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul6, indices_or_sections=2, axis=-1) split_01: R.Tensor((1, seq_len, 16384), dtype="float16") = split1[0] split_11: R.Tensor((1, seq_len, 16384), dtype="float16") = split1[1] gelu1: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_01) mul2: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu1, split_11) permute_dims7: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_1_mlp_down_proj_weight1, axes=None) matmul7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul2, permute_dims7, out_dtype="void") add3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul7, add2) rms_norm4: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add3, model_layers_2_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims8: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_2_self_attn_qkv_proj_weight1, axes=None) matmul8: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm4, permute_dims8, out_dtype="void") reshape8: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul8, R.shape([1, seq_len, 10, 256])) reshape9: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape8, R.shape([seq_len, 10, 256])) lv2 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape10: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv2, R.shape([1, seq_len, 8, 256])) reshape11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape10, R.shape([1, seq_len, 2048])) permute_dims9: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_2_self_attn_o_proj_weight1, axes=None) matmul9: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape11, permute_dims9, out_dtype="void") add4: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul9, add3) rms_norm5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add4, model_layers_2_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims10: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_2_mlp_gate_up_proj_weight1, axes=None) matmul10: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm5, permute_dims10, out_dtype="void") split2: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul10, indices_or_sections=2, axis=-1) split_02: R.Tensor((1, seq_len, 16384), dtype="float16") = split2[0] split_12: R.Tensor((1, seq_len, 16384), dtype="float16") = split2[1] gelu2: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_02) mul3: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu2, split_12) permute_dims11: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_2_mlp_down_proj_weight1, axes=None) matmul11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul3, permute_dims11, out_dtype="void") add5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul11, add4) rms_norm6: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add5, model_layers_3_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims12: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_3_self_attn_qkv_proj_weight1, axes=None) matmul12: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm6, permute_dims12, out_dtype="void") reshape12: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul12, R.shape([1, seq_len, 10, 256])) reshape13: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape12, R.shape([seq_len, 10, 256])) lv3 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape14: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv3, R.shape([1, seq_len, 8, 256])) reshape15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape14, R.shape([1, seq_len, 2048])) permute_dims13: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_3_self_attn_o_proj_weight1, axes=None) matmul13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape15, permute_dims13, out_dtype="void") add6: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul13, add5) rms_norm7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add6, model_layers_3_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims14: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_3_mlp_gate_up_proj_weight1, axes=None) matmul14: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm7, permute_dims14, out_dtype="void") split3: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul14, indices_or_sections=2, axis=-1) split_03: R.Tensor((1, seq_len, 16384), dtype="float16") = split3[0] split_13: R.Tensor((1, seq_len, 16384), dtype="float16") = split3[1] gelu3: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_03) mul4: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu3, split_13) permute_dims15: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_3_mlp_down_proj_weight1, axes=None) matmul15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul4, permute_dims15, out_dtype="void") add7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul15, add6) rms_norm8: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add7, model_layers_4_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims16: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_4_self_attn_qkv_proj_weight1, axes=None) matmul16: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm8, permute_dims16, out_dtype="void") reshape16: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul16, R.shape([1, seq_len, 10, 256])) reshape17: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape16, R.shape([seq_len, 10, 256])) lv4 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape18: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv4, R.shape([1, seq_len, 8, 256])) reshape19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape18, R.shape([1, seq_len, 2048])) permute_dims17: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_4_self_attn_o_proj_weight1, axes=None) matmul17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape19, permute_dims17, out_dtype="void") add8: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul17, add7) rms_norm9: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add8, model_layers_4_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims18: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_4_mlp_gate_up_proj_weight1, axes=None) matmul18: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm9, permute_dims18, out_dtype="void") split4: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul18, indices_or_sections=2, axis=-1) split_04: R.Tensor((1, seq_len, 16384), dtype="float16") = split4[0] split_14: R.Tensor((1, seq_len, 16384), dtype="float16") = split4[1] gelu4: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_04) mul5: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu4, split_14) permute_dims19: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_4_mlp_down_proj_weight1, axes=None) matmul19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul5, permute_dims19, out_dtype="void") add9: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul19, add8) rms_norm10: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add9, model_layers_5_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims20: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_5_self_attn_qkv_proj_weight1, axes=None) matmul20: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm10, permute_dims20, out_dtype="void") reshape20: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul20, R.shape([1, seq_len, 10, 256])) reshape21: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape20, R.shape([seq_len, 10, 256])) lv5 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape22: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv5, R.shape([1, seq_len, 8, 256])) reshape23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape22, R.shape([1, seq_len, 2048])) permute_dims21: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_5_self_attn_o_proj_weight1, axes=None) matmul21: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape23, permute_dims21, out_dtype="void") add10: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul21, add9) rms_norm11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add10, model_layers_5_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims22: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_5_mlp_gate_up_proj_weight1, axes=None) matmul22: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm11, permute_dims22, out_dtype="void") split5: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul22, indices_or_sections=2, axis=-1) split_05: R.Tensor((1, seq_len, 16384), dtype="float16") = split5[0] split_15: R.Tensor((1, seq_len, 16384), dtype="float16") = split5[1] gelu5: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_05) mul6: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu5, split_15) permute_dims23: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_5_mlp_down_proj_weight1, axes=None) matmul23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul6, permute_dims23, out_dtype="void") add11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul23, add10) rms_norm12: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add11, model_layers_6_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims24: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_6_self_attn_qkv_proj_weight1, axes=None) matmul24: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm12, permute_dims24, out_dtype="void") reshape24: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul24, R.shape([1, seq_len, 10, 256])) reshape25: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape24, R.shape([seq_len, 10, 256])) lv6 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape25), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape26: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv6, R.shape([1, seq_len, 8, 256])) reshape27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape26, R.shape([1, seq_len, 2048])) permute_dims25: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_6_self_attn_o_proj_weight1, axes=None) matmul25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape27, permute_dims25, out_dtype="void") add12: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul25, add11) rms_norm13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add12, model_layers_6_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims26: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_6_mlp_gate_up_proj_weight1, axes=None) matmul26: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm13, permute_dims26, out_dtype="void") split6: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul26, indices_or_sections=2, axis=-1) split_06: R.Tensor((1, seq_len, 16384), dtype="float16") = split6[0] split_16: R.Tensor((1, seq_len, 16384), dtype="float16") = split6[1] gelu6: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_06) mul7: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu6, split_16) permute_dims27: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_6_mlp_down_proj_weight1, axes=None) matmul27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul7, permute_dims27, out_dtype="void") add13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul27, add12) rms_norm14: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add13, model_layers_7_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims28: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_7_self_attn_qkv_proj_weight1, axes=None) matmul28: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm14, permute_dims28, out_dtype="void") reshape28: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul28, R.shape([1, seq_len, 10, 256])) reshape29: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape28, R.shape([seq_len, 10, 256])) lv7 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape29), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape30: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv7, R.shape([1, seq_len, 8, 256])) reshape31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape30, R.shape([1, seq_len, 2048])) permute_dims29: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_7_self_attn_o_proj_weight1, axes=None) matmul29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape31, permute_dims29, out_dtype="void") add14: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul29, add13) rms_norm15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add14, model_layers_7_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims30: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_7_mlp_gate_up_proj_weight1, axes=None) matmul30: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm15, permute_dims30, out_dtype="void") split7: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul30, indices_or_sections=2, axis=-1) split_07: R.Tensor((1, seq_len, 16384), dtype="float16") = split7[0] split_17: R.Tensor((1, seq_len, 16384), dtype="float16") = split7[1] gelu7: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_07) mul8: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu7, split_17) permute_dims31: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_7_mlp_down_proj_weight1, axes=None) matmul31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul8, permute_dims31, out_dtype="void") add15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul31, add14) rms_norm16: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add15, model_layers_8_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims32: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_8_self_attn_qkv_proj_weight1, axes=None) matmul32: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm16, permute_dims32, out_dtype="void") reshape32: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul32, R.shape([1, seq_len, 10, 256])) reshape33: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape32, R.shape([seq_len, 10, 256])) lv8 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape33), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape34: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv8, R.shape([1, seq_len, 8, 256])) reshape35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape34, R.shape([1, seq_len, 2048])) permute_dims33: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_8_self_attn_o_proj_weight1, axes=None) matmul33: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape35, permute_dims33, out_dtype="void") add16: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul33, add15) rms_norm17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add16, model_layers_8_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims34: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_8_mlp_gate_up_proj_weight1, axes=None) matmul34: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm17, permute_dims34, out_dtype="void") split8: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul34, indices_or_sections=2, axis=-1) split_08: R.Tensor((1, seq_len, 16384), dtype="float16") = split8[0] split_18: R.Tensor((1, seq_len, 16384), dtype="float16") = split8[1] gelu8: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_08) mul9: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu8, split_18) permute_dims35: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_8_mlp_down_proj_weight1, axes=None) matmul35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul9, permute_dims35, out_dtype="void") add17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul35, add16) rms_norm18: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add17, model_layers_9_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims36: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_9_self_attn_qkv_proj_weight1, axes=None) matmul36: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm18, permute_dims36, out_dtype="void") reshape36: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul36, R.shape([1, seq_len, 10, 256])) reshape37: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape36, R.shape([seq_len, 10, 256])) lv9 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape37), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape38: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv9, R.shape([1, seq_len, 8, 256])) reshape39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape38, R.shape([1, seq_len, 2048])) permute_dims37: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_9_self_attn_o_proj_weight1, axes=None) matmul37: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape39, permute_dims37, out_dtype="void") add18: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul37, add17) rms_norm19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add18, model_layers_9_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims38: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_9_mlp_gate_up_proj_weight1, axes=None) matmul38: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm19, permute_dims38, out_dtype="void") split9: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul38, indices_or_sections=2, axis=-1) split_09: R.Tensor((1, seq_len, 16384), dtype="float16") = split9[0] split_19: R.Tensor((1, seq_len, 16384), dtype="float16") = split9[1] gelu9: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_09) mul10: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu9, split_19) permute_dims39: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_9_mlp_down_proj_weight1, axes=None) matmul39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul10, permute_dims39, out_dtype="void") add19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul39, add18) rms_norm20: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add19, model_layers_10_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims40: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_10_self_attn_qkv_proj_weight1, axes=None) matmul40: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm20, permute_dims40, out_dtype="void") reshape40: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul40, R.shape([1, seq_len, 10, 256])) reshape41: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape40, R.shape([seq_len, 10, 256])) lv10 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape41), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape42: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv10, R.shape([1, seq_len, 8, 256])) reshape43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape42, R.shape([1, seq_len, 2048])) permute_dims41: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_10_self_attn_o_proj_weight1, axes=None) matmul41: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape43, permute_dims41, out_dtype="void") add20: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul41, add19) rms_norm21: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add20, model_layers_10_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims42: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_10_mlp_gate_up_proj_weight1, axes=None) matmul42: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm21, permute_dims42, out_dtype="void") split10: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul42, indices_or_sections=2, axis=-1) split_010: R.Tensor((1, seq_len, 16384), dtype="float16") = split10[0] split_110: R.Tensor((1, seq_len, 16384), dtype="float16") = split10[1] gelu10: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_010) mul11: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu10, split_110) permute_dims43: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_10_mlp_down_proj_weight1, axes=None) matmul43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul11, permute_dims43, out_dtype="void") add21: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul43, add20) rms_norm22: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add21, model_layers_11_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims44: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_11_self_attn_qkv_proj_weight1, axes=None) matmul44: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm22, permute_dims44, out_dtype="void") reshape44: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul44, R.shape([1, seq_len, 10, 256])) reshape45: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape44, R.shape([seq_len, 10, 256])) lv11 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape45), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape46: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv11, R.shape([1, seq_len, 8, 256])) reshape47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape46, R.shape([1, seq_len, 2048])) permute_dims45: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_11_self_attn_o_proj_weight1, axes=None) matmul45: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape47, permute_dims45, out_dtype="void") add22: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul45, add21) rms_norm23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add22, model_layers_11_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims46: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_11_mlp_gate_up_proj_weight1, axes=None) matmul46: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm23, permute_dims46, out_dtype="void") split11: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul46, indices_or_sections=2, axis=-1) split_011: R.Tensor((1, seq_len, 16384), dtype="float16") = split11[0] split_111: R.Tensor((1, seq_len, 16384), dtype="float16") = split11[1] gelu11: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_011) mul12: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu11, split_111) permute_dims47: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_11_mlp_down_proj_weight1, axes=None) matmul47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul12, permute_dims47, out_dtype="void") add23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul47, add22) rms_norm24: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add23, model_layers_12_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims48: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_12_self_attn_qkv_proj_weight1, axes=None) matmul48: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm24, permute_dims48, out_dtype="void") reshape48: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul48, R.shape([1, seq_len, 10, 256])) reshape49: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape48, R.shape([seq_len, 10, 256])) lv12 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape49), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape50: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv12, R.shape([1, seq_len, 8, 256])) reshape51: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape50, R.shape([1, seq_len, 2048])) permute_dims49: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_12_self_attn_o_proj_weight1, axes=None) matmul49: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape51, permute_dims49, out_dtype="void") add24: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul49, add23) rms_norm25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add24, model_layers_12_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims50: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_12_mlp_gate_up_proj_weight1, axes=None) matmul50: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm25, permute_dims50, out_dtype="void") split12: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul50, indices_or_sections=2, axis=-1) split_012: R.Tensor((1, seq_len, 16384), dtype="float16") = split12[0] split_112: R.Tensor((1, seq_len, 16384), dtype="float16") = split12[1] gelu12: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_012) mul13: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu12, split_112) permute_dims51: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_12_mlp_down_proj_weight1, axes=None) matmul51: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul13, permute_dims51, out_dtype="void") add25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul51, add24) rms_norm26: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add25, model_layers_13_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims52: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_13_self_attn_qkv_proj_weight1, axes=None) matmul52: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm26, permute_dims52, out_dtype="void") reshape52: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul52, R.shape([1, seq_len, 10, 256])) reshape53: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape52, R.shape([seq_len, 10, 256])) lv13 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape53), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape54: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv13, R.shape([1, seq_len, 8, 256])) reshape55: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape54, R.shape([1, seq_len, 2048])) permute_dims53: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_13_self_attn_o_proj_weight1, axes=None) matmul53: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape55, permute_dims53, out_dtype="void") add26: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul53, add25) rms_norm27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add26, model_layers_13_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims54: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_13_mlp_gate_up_proj_weight1, axes=None) matmul54: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm27, permute_dims54, out_dtype="void") split13: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul54, indices_or_sections=2, axis=-1) split_013: R.Tensor((1, seq_len, 16384), dtype="float16") = split13[0] split_113: R.Tensor((1, seq_len, 16384), dtype="float16") = split13[1] gelu13: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_013) mul14: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu13, split_113) permute_dims55: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_13_mlp_down_proj_weight1, axes=None) matmul55: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul14, permute_dims55, out_dtype="void") add27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul55, add26) rms_norm28: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add27, model_layers_14_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims56: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_14_self_attn_qkv_proj_weight1, axes=None) matmul56: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm28, permute_dims56, out_dtype="void") reshape56: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul56, R.shape([1, seq_len, 10, 256])) reshape57: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape56, R.shape([seq_len, 10, 256])) lv14 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape57), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape58: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv14, R.shape([1, seq_len, 8, 256])) reshape59: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape58, R.shape([1, seq_len, 2048])) permute_dims57: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_14_self_attn_o_proj_weight1, axes=None) matmul57: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape59, permute_dims57, out_dtype="void") add28: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul57, add27) rms_norm29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add28, model_layers_14_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims58: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_14_mlp_gate_up_proj_weight1, axes=None) matmul58: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm29, permute_dims58, out_dtype="void") split14: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul58, indices_or_sections=2, axis=-1) split_014: R.Tensor((1, seq_len, 16384), dtype="float16") = split14[0] split_114: R.Tensor((1, seq_len, 16384), dtype="float16") = split14[1] gelu14: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_014) mul15: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu14, split_114) permute_dims59: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_14_mlp_down_proj_weight1, axes=None) matmul59: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul15, permute_dims59, out_dtype="void") add29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul59, add28) rms_norm30: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add29, model_layers_15_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims60: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_15_self_attn_qkv_proj_weight1, axes=None) matmul60: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm30, permute_dims60, out_dtype="void") reshape60: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul60, R.shape([1, seq_len, 10, 256])) reshape61: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape60, R.shape([seq_len, 10, 256])) lv15 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape61), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape62: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv15, R.shape([1, seq_len, 8, 256])) reshape63: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape62, R.shape([1, seq_len, 2048])) permute_dims61: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_15_self_attn_o_proj_weight1, axes=None) matmul61: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape63, permute_dims61, out_dtype="void") add30: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul61, add29) rms_norm31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add30, model_layers_15_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims62: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_15_mlp_gate_up_proj_weight1, axes=None) matmul62: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm31, permute_dims62, out_dtype="void") split15: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul62, indices_or_sections=2, axis=-1) split_015: R.Tensor((1, seq_len, 16384), dtype="float16") = split15[0] split_115: R.Tensor((1, seq_len, 16384), dtype="float16") = split15[1] gelu15: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_015) mul16: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu15, split_115) permute_dims63: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_15_mlp_down_proj_weight1, axes=None) matmul63: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul16, permute_dims63, out_dtype="void") add31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul63, add30) rms_norm32: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add31, model_layers_16_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims64: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_16_self_attn_qkv_proj_weight1, axes=None) matmul64: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm32, permute_dims64, out_dtype="void") reshape64: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul64, R.shape([1, seq_len, 10, 256])) reshape65: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape64, R.shape([seq_len, 10, 256])) lv16 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape65), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape66: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv16, R.shape([1, seq_len, 8, 256])) reshape67: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape66, R.shape([1, seq_len, 2048])) permute_dims65: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_16_self_attn_o_proj_weight1, axes=None) matmul65: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape67, permute_dims65, out_dtype="void") add32: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul65, add31) rms_norm33: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add32, model_layers_16_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims66: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_16_mlp_gate_up_proj_weight1, axes=None) matmul66: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm33, permute_dims66, out_dtype="void") split16: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul66, indices_or_sections=2, axis=-1) split_016: R.Tensor((1, seq_len, 16384), dtype="float16") = split16[0] split_116: R.Tensor((1, seq_len, 16384), dtype="float16") = split16[1] gelu16: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_016) mul17: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu16, split_116) permute_dims67: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_16_mlp_down_proj_weight1, axes=None) matmul67: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul17, permute_dims67, out_dtype="void") add33: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul67, add32) rms_norm34: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add33, model_layers_17_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims68: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_17_self_attn_qkv_proj_weight1, axes=None) matmul68: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm34, permute_dims68, out_dtype="void") reshape68: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(matmul68, R.shape([1, seq_len, 10, 256])) reshape69: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape68, R.shape([seq_len, 10, 256])) lv17 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape69), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape70: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv17, R.shape([1, seq_len, 8, 256])) reshape71: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape70, R.shape([1, seq_len, 2048])) permute_dims69: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_17_self_attn_o_proj_weight1, axes=None) matmul69: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape71, permute_dims69, out_dtype="void") add34: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul69, add33) rms_norm35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add34, model_layers_17_post_attention_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) permute_dims70: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_17_mlp_gate_up_proj_weight1, axes=None) matmul70: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm35, permute_dims70, out_dtype="void") split17: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(matmul70, indices_or_sections=2, axis=-1) split_017: R.Tensor((1, seq_len, 16384), dtype="float16") = split17[0] split_117: R.Tensor((1, seq_len, 16384), dtype="float16") = split17[1] gelu17: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_017) mul18: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu17, split_117) permute_dims71: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_17_mlp_down_proj_weight1, axes=None) matmul71: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul18, permute_dims71, out_dtype="void") add35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul71, add34) rms_norm36: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(add35, model_norm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) lv18 = R.call_tir(cls.index, (rms_norm36,), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) permute_dims72: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight1, axes=None) matmul72: R.Tensor((1, 1, vocab_size), dtype="float32") = R.matmul(lv18, permute_dims72, out_dtype="float32") gv1: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = matmul72, paged_kv_cache R.output(gv1) return gv1 @R.function def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64(is_size_var=True) vocab_size = T.int64(is_size_var=True) R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", logits, R.shape([batch_size, vocab_size]), sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) lv1 = R.call_tir(cls.chunk_lse, (lv, temperature), out_sinfo=[R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32"), R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32")]) lv2: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[0] lv3: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[1] lv4 = R.call_tir(cls.softmax_with_chunked_sum, (lv, temperature, lv2, lv3), out_sinfo=R.Tensor((batch_size, vocab_size), dtype="float32")) gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", lv4, R.shape([batch_size, 1, vocab_size]), sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) R.output(gv) return gv