# from tvm.script import ir as I # from tvm.script import tir as T # from tvm.script import relax as R @I.ir_module class Module: I.module_attrs({"system_lib_prefix": "gpt_neox_q4f16_1_"}) @T.prim_func def gpt_neox_q4f16_1_apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") # with T.block("root"): for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) T.writes(logits[seq_ids[vs], vv]) logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) @T.prim_func def gpt_neox_q4f16_1_apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") logit_bias = T.match_buffer(var_logit_bias, (num_token,)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) T.writes(logits[pos2seq_id[vp], token_ids[vp]]) logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] @T.prim_func def gpt_neox_q4f16_1_apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") penalties = T.match_buffer(var_penalties, (num_seq, 3)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) @T.prim_func def gpt_neox_q4f16_1_batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 64), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 64), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.18033688011112042) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"): for ty in T.thread_binding(1, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): for tz in T.thread_binding(16, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68]) T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared") md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((1,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = fused_by_bz % 8 bz: T.int32 = fused_by_bz // 8 batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 15) // 16): tile_start_s: T.int32 = tz + ty tile_start_g: T.int32 = iterator * 16 + tz + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = row_g page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(1): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if iterator * 16 + tz + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(1): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(1): for vec in T.vectorized(4): V_local[vec] = V_smem[tz + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] md_allreduce[tz, ty, 0] = st_m[0] md_allreduce[tz, ty, 1] = st_d[0] T.tvm_storage_sync("shared") st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for j in range(16): m_prev[0] = st_m[0] d_prev[0] = st_d[0] other_m[0] = md_allreduce[j, ty, 0] other_d[0] = md_allreduce[j, ty, 1] for vec in T.vectorized(4): other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] st_m[0] = T.max(st_m[0], other_m[0]) st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) for vec in T.vectorized(4): O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 64), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 64), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.18033688011112042) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"): for ty in T.thread_binding(1, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): for tz in T.thread_binding(16, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68]) T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared") md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((1,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = fused_by_bz % 8 bz: T.int32 = fused_by_bz // 8 batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 15) // 16): tile_start_s: T.int32 = tz + ty tile_start_g: T.int32 = iterator * 16 + tz + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(1): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if iterator * 16 + tz + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(1): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(1): for vec in T.vectorized(4): V_local[vec] = V_smem[tz + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] md_allreduce[tz, ty, 0] = st_m[0] md_allreduce[tz, ty, 1] = st_d[0] T.tvm_storage_sync("shared") st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for j in range(16): m_prev[0] = st_m[0] d_prev[0] = st_d[0] other_m[0] = md_allreduce[j, ty, 0] other_d[0] = md_allreduce[j, ty, 1] for vec in T.vectorized(4): other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] st_m[0] = T.max(st_m[0], other_m[0]) st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) for vec in T.vectorized(4): O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def gpt_neox_q4f16_1_batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 16), scope="shared") S_local = T.alloc_buffer((32, 16), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64]) T.writes(S_local[0:32, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 16), scope="shared") S_local = T.alloc_buffer((32, 16), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64]) T.writes(S_local[0:32, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gpt_neox_q4f16_1_batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 8, 64), "float16") v = T.match_buffer(var_v, (kv_len, 8, 64), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(8, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((64, 32), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] q_indptr_val: T.int32 = q_indptr[b_idx] LH_start: T.int32 = tile_id[0] * 32 kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_0, lj_0_0 in T.grid(1, 2): for lj_1 in T.vectorized(8): with T.block("Q_load"): i = T.axis.spatial(32, li_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 4) j = T.axis.spatial(64, lj_0_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 4 * 8 + lj_1) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(1, 2): for ly_1 in T.vectorized(8): with T.block("K_load"): i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4) j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1) T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 32:j - 32 + 65]) T.writes(K_smem[j, i]) cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: freq = T.float32() K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[L_kv_base + cur_L, by, j + 32] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[L_kv_base + cur_L, by, j]) else: K_smem[j, i] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(1, 2): for ly_1 in T.vectorized(8): with T.block("V_load"): i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4) j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: V_smem[i, j] = v[L_kv_base + cur_L, by, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:64, 0:32]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(2): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_0_init * 4 + lj_1_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0 in range(4): for li_1 in T.unroll(2): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): for lk_1 in range(16): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) k_1 = T.axis.reduce(64, lk_0 * 16 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(2): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(4): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_0_init * 4 + lj_1_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1 in T.grid(2, 16): for li_1 in T.unroll(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) k_1 = T.axis.reduce(32, lk_0 * 16 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 8 @T.prim_func def gpt_neox_q4f16_1_batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 64), "float16") q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 8, 64), "float16") v = T.match_buffer(var_v, (kv_len, 8, 64), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) tree_size = T.int32(is_size_var=True) mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((32, 64), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("KV_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_base + L_kv_start + i if L_kv_start + i < kv_chunk_len[0]: freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[cur_L, by, j + 32] * T.float16(-1.0), k[cur_L, by, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[cur_L, by, j]) V_smem[i, j] = v[cur_L, by, j] else: K_smem[i, j] = T.float16(0.0) V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) k_1 = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gpt_neox_q4f16_1_chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int32(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size, num_chunks), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size, num_chunks), scope="shared") for ax0_ax1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0, ax1 in T.grid(1, 1): for ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks + ax0) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks + ax1) v2 = T.axis.reduce(4096, ax2_fused_0 * 256 + ax2_fused_1) T.reads(temperature[v0], A[v0, v1 * 4096 + v2]) T.writes(temp_max_shared[v0, v1]) with T.init(): temp_max_shared[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0, v1] = T.max(temp_max_shared[v0, v1], T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0))) for ax0, ax1 in T.grid(1, 1): for ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks + ax0) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks + ax1) v2 = T.axis.reduce(4096, ax2_fused_0 * 256 + ax2_fused_1) T.reads(temperature[v0], A[v0, v1 * 4096 + v2], temp_max_shared[v0, v1]) T.writes(temp_sum_shared[v0, v1]) with T.init(): temp_sum_shared[v0, v1] = T.float32(0.0) temp_sum_shared[v0, v1] = temp_sum_shared[v0, v1] + T.if_then_else(v1 * 4096 + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0)) - temp_max_shared[v0, v1]), T.Cast("float32", T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0)) == temp_max_shared[v0, v1])), T.float32(0.0)) for ax2_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("log"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks) v2 = T.axis.spatial(1, ax2_0 * 256 + ax2_1) T.where(ax2_0 * 256 + ax2_1 < 1) T.reads(temperature[v0], temp_sum_shared[v0, v1], temp_max_shared[v0, v1]) T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum_shared[v0, v1]), temp_sum_shared[v0, v1]) chunked_max[v0, v1] = temp_max_shared[v0, v1] @T.prim_func def gpt_neox_q4f16_1_compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 8, 16, 64), "float16", offset_factor=1) copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) total_copy_length = T.int32() copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) with T.block("root"): T.reads() T.writes() for bhd_o in T.thread_binding(batch_size * 2, thread="blockIdx.x"): for bhd_i in T.thread_binding(256, thread="threadIdx.x"): b: T.int32 = (bhd_o * 256 + bhd_i) // 512 h: T.int32 = (bhd_o * 256 + bhd_i) // 64 % 8 d: T.int32 = (bhd_o * 256 + bhd_i) % 64 if bhd_o * 256 + bhd_i < batch_size * 8 * 64: for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] @T.prim_func def gpt_neox_q4f16_1_copy_single_page(var_pages: T.handle, src_page_id: T.int32, tgt_page_id: T.int32, copy_length: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages, page_size = T.int32(), T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 8, page_size, 64), "float16", offset_factor=1) # with T.block("root"): for b in T.thread_binding(copy_length * 2, thread="blockIdx.x"): for t in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy"): vh = T.axis.spatial(8, (b * 256 + t) // (copy_length * 64)) vp = T.axis.spatial(copy_length, (b * 256 + t) % (copy_length * 64) // 64) vd = T.axis.spatial(64, (b * 256 + t) % 64) T.where(b * 256 + t < copy_length * 8 * 64) T.reads(pages[src_page_id, 0:2, vh, vp, vd]) T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_0_attention_query_key_value_q_weight2: T.Buffer((1536, 64), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale2: T.Buffer((1536, 16), "float16"), layer_norm13: T.Buffer((1, 1, 512), "float16"), gpt_neox_layers_0_attention_query_key_value_bias2: T.Buffer((1536,), "float16"), T_add_intermediate_intermediate: T.Buffer((1, 1, 1536), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 1536), "float16", scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 1536), "float16", scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 1536), "float16", scope="local") gpt_neox_layers_0_attention_query_key_value_q_weight2_local = T.alloc_buffer((1536, 64), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(768, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_attention_query_key_value_q_weight2_local"): v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(64, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, v1]) gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, v1] = gpt_neox_layers_0_attention_query_key_value_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], layer_norm13[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_attention_query_key_value_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + layer_norm13[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused] * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float16(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("T_add"): v0 = T.axis.spatial(1536, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_attention_query_key_value_bias2[v0]) T.writes(T_add_intermediate_intermediate[0, 0, v0]) T_add_intermediate_intermediate[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_attention_query_key_value_bias2[v0] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight3: T.Buffer((1536, 64), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale3: T.Buffer((1536, 16), "float16"), p_layer_norm26: T.handle, gpt_neox_layers_0_attention_query_key_value_bias3: T.Buffer((1536,), "float16"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm26 = T.match_buffer(p_layer_norm26, (1, seq_len, 512), "float16") T_add_intermediate_intermediate = T.match_buffer(p_output0, (1, seq_len, 1536), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1536, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 1536), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(384, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, layer_norm26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_query_key_value_bias3[v1]) T.writes(T_add_intermediate_intermediate[0, v0, v1]) T_add_intermediate_intermediate[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_query_key_value_bias3[v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1536, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 1536), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(384, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, layer_norm26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_query_key_value_bias3[v1]) T.writes(T_add_intermediate_intermediate[0, v0, v1]) T_add_intermediate_intermediate[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_query_key_value_bias3[v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 1536), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(6, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(1536, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(1536, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm26[0, v1, v3], gpt_neox_layers_0_attention_query_key_value_q_weight3[v2, v3 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, layer_norm26[0, v1, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(1536, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_query_key_value_bias3[v2]) T.writes(T_add_intermediate_intermediate[0, v1, v2]) T_add_intermediate_intermediate[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_query_key_value_bias3[v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_0_attention_query_key_value_q_weight4: T.Buffer((1536, 64), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale4: T.Buffer((1536, 16), "float16"), p_layer_norm39: T.handle, gpt_neox_layers_0_attention_query_key_value_bias4: T.Buffer((1536,), "float16"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() layer_norm39 = T.match_buffer(p_layer_norm39, (batch_size, 1, 512), "float16") T_add_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, 1536), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1536, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 1536), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(384, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm39[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, layer_norm39[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_query_key_value_bias4[v1]) T.writes(T_add_intermediate_intermediate[v0, 0, v1]) T_add_intermediate_intermediate[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_query_key_value_bias4[v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1536, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 1536), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 1536), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(384, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm39[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, layer_norm39[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(1536, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_query_key_value_bias4[v1]) T.writes(T_add_intermediate_intermediate[v0, 0, v1]) T_add_intermediate_intermediate[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_query_key_value_bias4[v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 1536), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(6, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(1536, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(1536, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm39[v1, 0, v3], gpt_neox_layers_0_attention_query_key_value_q_weight4[v2, v3 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, layer_norm39[v1, 0, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(1536, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_query_key_value_bias4[v2]) T.writes(T_add_intermediate_intermediate[v1, 0, v2]) T_add_intermediate_intermediate[v1, 0, v2] = NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_query_key_value_bias4[v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_0_attention_dense_q_weight2: T.Buffer((512, 64), "uint32"), gpt_neox_layers_0_attention_dense_q_scale2: T.Buffer((512, 16), "float16"), lv77: T.Buffer((1, 1, 512), "float16"), gpt_neox_layers_0_attention_dense_bias2: T.Buffer((512,), "float16"), lv56: T.Buffer((1, 1, 512), "float16"), input_embed: T.Buffer((1, 1, 512), "float16"), T_add_intermediate_1_2_intermediate: T.Buffer((1, 1, 512), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 512), "float16", scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 512), "float16", scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 512), "float16", scope="local") gpt_neox_layers_0_attention_dense_q_weight2_local = T.alloc_buffer((512, 64), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_attention_dense_q_weight2_local"): v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(64, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_attention_dense_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_attention_dense_q_weight2_local[v0, v1]) gpt_neox_layers_0_attention_dense_q_weight2_local[v0, v1] = gpt_neox_layers_0_attention_dense_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], lv77[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_attention_dense_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_attention_dense_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + lv77[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused] * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float16(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("T_add_2"): v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(lv56[0, 0, v0], NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_attention_dense_bias2[v0], input_embed[0, 0, v0]) T.writes(T_add_intermediate_1_2_intermediate[0, 0, v0]) T_add_intermediate_1_2_intermediate[0, 0, v0] = lv56[0, 0, v0] + (NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_attention_dense_bias2[v0]) + input_embed[0, 0, v0] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_0_attention_dense_q_weight4: T.Buffer((512, 64), "uint32"), gpt_neox_layers_0_attention_dense_q_scale4: T.Buffer((512, 16), "float16"), p_reshape75: T.handle, gpt_neox_layers_0_attention_dense_bias4: T.Buffer((512,), "float16"), p_lv2: T.handle, p_input_embeds: T.handle, p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape75 = T.match_buffer(p_reshape75, (batch_size, 1, 512), "float16") lv2 = T.match_buffer(p_lv2, (batch_size, 1, 512), "float16") input_embeds = T.match_buffer(p_input_embeds, (batch_size, 1, 512), "float16") T_add_intermediate_1_2_intermediate = T.match_buffer(p_output0, (batch_size, 1, 512), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 512), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], reshape75[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, reshape75[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(lv2[v0, 0, v1], NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_dense_bias4[v1], input_embeds[v0, 0, v1]) T.writes(T_add_intermediate_1_2_intermediate[v0, 0, v1]) T_add_intermediate_1_2_intermediate[v0, 0, v1] = lv2[v0, 0, v1] + (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_dense_bias4[v1]) + input_embeds[v0, 0, v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 512), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], reshape75[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, reshape75[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(lv2[v0, 0, v1], NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_dense_bias4[v1], input_embeds[v0, 0, v1]) T.writes(T_add_intermediate_1_2_intermediate[v0, 0, v1]) T_add_intermediate_1_2_intermediate[v0, 0, v1] = lv2[v0, 0, v1] + (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_dense_bias4[v1]) + input_embeds[v0, 0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 512), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(2, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], reshape75[v1, 0, v3], gpt_neox_layers_0_attention_dense_q_weight4[v2, v3 // 8], gpt_neox_layers_0_attention_dense_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, reshape75[v1, 0, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(lv2[v1, 0, v2], NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_dense_bias4[v2], input_embeds[v1, 0, v2]) T.writes(T_add_intermediate_1_2_intermediate[v1, 0, v2]) T_add_intermediate_1_2_intermediate[v1, 0, v2] = lv2[v1, 0, v2] + (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_dense_bias4[v2]) + input_embeds[v1, 0, v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight3: T.Buffer((512, 64), "uint32"), gpt_neox_layers_0_attention_dense_q_scale3: T.Buffer((512, 16), "float16"), p_reshape51: T.handle, gpt_neox_layers_0_attention_dense_bias3: T.Buffer((512,), "float16"), p_lv20: T.handle, p_input_embeds: T.handle, p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape51 = T.match_buffer(p_reshape51, (1, seq_len, 512), "float16") lv20 = T.match_buffer(p_lv20, (1, seq_len, 512), "float16") input_embeds = T.match_buffer(p_input_embeds, (1, seq_len, 512), "float16") T_add_intermediate_1_2_intermediate = T.match_buffer(p_output0, (1, seq_len, 512), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 512), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], reshape51[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, reshape51[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(lv20[0, v0, v1], NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_dense_bias3[v1], input_embeds[0, v0, v1]) T.writes(T_add_intermediate_1_2_intermediate[0, v0, v1]) T_add_intermediate_1_2_intermediate[0, v0, v1] = lv20[0, v0, v1] + (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_dense_bias3[v1]) + input_embeds[0, v0, v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 512), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 512), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], reshape51[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, reshape51[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(lv20[0, v0, v1], NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_dense_bias3[v1], input_embeds[0, v0, v1]) T.writes(T_add_intermediate_1_2_intermediate[0, v0, v1]) T_add_intermediate_1_2_intermediate[0, v0, v1] = lv20[0, v0, v1] + (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_dense_bias3[v1]) + input_embeds[0, v0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 512), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(2, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], reshape51[0, v1, v3], gpt_neox_layers_0_attention_dense_q_weight3[v2, v3 // 8], gpt_neox_layers_0_attention_dense_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, reshape51[0, v1, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(lv20[0, v1, v2], NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_dense_bias3[v2], input_embeds[0, v1, v2]) T.writes(T_add_intermediate_1_2_intermediate[0, v1, v2]) T_add_intermediate_1_2_intermediate[0, v1, v2] = lv20[0, v1, v2] + (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_dense_bias3[v2]) + input_embeds[0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2: T.Buffer((2048, 64), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2: T.Buffer((2048, 16), "float16"), layer_norm14: T.Buffer((1, 1, 512), "float16"), gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: T.Buffer((2048,), "float32"), compute_intermediate_intermediate: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 2048), scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 2048), scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 2048), scope="local") gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local = T.alloc_buffer((2048, 64), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float32(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local"): v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(64, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, v1]) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, v1] = gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], layer_norm14[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + T.Cast("float32", layer_norm14[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused]) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float32(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("compute_1_1"): v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v0]) T.writes(compute_intermediate_intermediate[0, 0, v0]) compute_intermediate_intermediate[0, 0, v0] = T.Cast("float16", (NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v0]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v0]) * T.float32(0.70710678118654757)) * T.float32(0.5))) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4: T.Buffer((2048, 64), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4: T.Buffer((2048, 16), "float16"), p_layer_norm40: T.handle, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: T.Buffer((2048,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() layer_norm40 = T.match_buffer(p_layer_norm40, (batch_size, 1, 512), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, 2048), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((2048, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 2048), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 2048), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 2048), scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm40[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, layer_norm40[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((2048, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 2048), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 2048), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 2048), scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm40[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, layer_norm40[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 2048), scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(8, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm40[v1, 0, v3], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < batch_size, layer_norm40[v1, 0, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v2]) T.writes(compute_intermediate_intermediate[v1, 0, v2]) compute_intermediate_intermediate[v1, 0, v2] = T.Cast("float16", (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v2]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v2]) * T.float32(0.70710678118654757)) * T.float32(0.5))) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3: T.Buffer((2048, 64), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3: T.Buffer((2048, 16), "float16"), p_layer_norm27: T.handle, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: T.Buffer((2048,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm27 = T.match_buffer(p_layer_norm27, (1, seq_len, 512), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, seq_len, 2048), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((2048, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 2048), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 2048), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 2048), scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm27[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, layer_norm27[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((2048, 512), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 2048), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 2048), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 2048), scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(512, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm27[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, layer_norm27[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 2048), scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(8, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm27[0, v1, v3], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < seq_len, layer_norm27[0, v1, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v2]) T.writes(compute_intermediate_intermediate[0, v1, v2]) compute_intermediate_intermediate[0, v1, v2] = T.Cast("float16", (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v2]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v2]) * T.float32(0.70710678118654757)) * T.float32(0.5))) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2: T.Buffer((512, 256), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2: T.Buffer((512, 64), "float16"), lv78: T.Buffer((1, 1, 2048), "float16"), gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: T.Buffer((512,), "float32"), compute_intermediate_intermediate: T.Buffer((1, 1, 512), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 512), scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 512), scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 512), scope="local") gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local = T.alloc_buffer((512, 256), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float32(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local"): v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, v1]) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, v1] = gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], lv78[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + T.Cast("float32", lv78[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused]) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float32(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("compute_1"): v0 = T.axis.spatial(512, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[v0]) T.writes(compute_intermediate_intermediate[0, 0, v0]) compute_intermediate_intermediate[0, 0, v0] = T.Cast("float16", NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[v0]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4: T.Buffer((512, 256), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4: T.Buffer((512, 64), "float16"), p_lv1: T.handle, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: T.Buffer((512,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv1 = T.match_buffer(p_lv1, (batch_size, 1, 2048), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, 512), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 2048), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 512), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 512), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 512), scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(2048, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 2048), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 512), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 512), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 512), scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(2048, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 512), scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(2, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], lv1[v1, 0, v3], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < batch_size, lv1[v1, 0, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v2]) T.writes(compute_intermediate_intermediate[v1, 0, v2]) compute_intermediate_intermediate[v1, 0, v2] = T.Cast("float16", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3: T.Buffer((512, 256), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3: T.Buffer((512, 64), "float16"), p_lv26: T.handle, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: T.Buffer((512,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv26 = T.match_buffer(p_lv26, (1, seq_len, 2048), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, seq_len, 512), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 2048), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 512), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 512), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 512), scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(2048, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], lv26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, lv26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((512, 2048), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 512), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 512), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 512), scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(2048, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], lv26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, lv26[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(512, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 512), scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(2, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], lv26[0, v1, v3], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < seq_len, lv26[0, v1, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(512, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v2]) T.writes(compute_intermediate_intermediate[0, v1, v2]) compute_intermediate_intermediate[0, v1, v2] = T.Cast("float16", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8(p_embed_out_q_weight2: T.handle, p_embed_out_q_scale2: T.handle, layer_norm25: T.Buffer((1, 1, 512), "float16"), p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() embed_out_q_weight2 = T.match_buffer(p_embed_out_q_weight2, (vocab_size, 64), "uint32") embed_out_q_scale2 = T.match_buffer(p_embed_out_q_scale2, (vocab_size, 16), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, 1, vocab_size)) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, vocab_size), "float16", scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, vocab_size), "float16", scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, vocab_size), "float16", scope="local") embed_out_q_weight2_local = T.alloc_buffer((vocab_size, 64), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding((vocab_size + 1) // 2, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init < vocab_size) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("embed_out_q_weight2_local"): v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(64, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 < vocab_size) T.reads(embed_out_q_weight2[v0, v1]) T.writes(embed_out_q_weight2_local[v0, v1]) embed_out_q_weight2_local[v0, v1] = embed_out_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2 < vocab_size) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], layer_norm25[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], embed_out_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], embed_out_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + layer_norm25[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused] * ((T.Cast("float16", T.bitwise_and(T.shift_right(embed_out_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * embed_out_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax2_fused_0_ax2_fused_1_fused % 2 + (ax2_fused_2_0 + ax2_fused_2_1)) < vocab_size) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax2_fused_0_ax2_fused_1_fused % 2 + (ax2_fused_2_0 + ax2_fused_2_1)) < vocab_size) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax1_fused_0_ax1_fused_1_fused % 2 + ax1_fused_2) < vocab_size) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float16(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("compute_1"): v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax0_fused_0_ax0_fused_1_fused % 2 + ax0_fused_2) < vocab_size) T.reads(NT_matmul_intermediate_local[0, 0, v0]) T.writes(compute_intermediate_intermediate[0, 0, v0]) compute_intermediate_intermediate[0, 0, v0] = T.Cast("float32", NT_matmul_intermediate_local[0, 0, v0]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2(p_embed_out_q_weight4: T.handle, p_embed_out_q_scale4: T.handle, p_layer_norm51: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() embed_out_q_weight4 = T.match_buffer(p_embed_out_q_weight4, (vocab_size, 64), "uint32") embed_out_q_scale4 = T.match_buffer(p_embed_out_q_scale4, (vocab_size, 16), "float16") batch_size = T.int32() layer_norm51 = T.match_buffer(p_layer_norm51, (batch_size, 1, 512), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, vocab_size)) # with T.block("root"): NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, (vocab_size + 255) // 256 * 256), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding((vocab_size + 255) // 256, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm51[v1, 0, v3], embed_out_q_weight4[v2, v3 // 8], embed_out_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, layer_norm51[v1, 0, v3], T.float16(0.0)) * T.if_then_else(v2 < vocab_size, (T.Cast("float16", T.bitwise_and(T.shift_right(embed_out_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * embed_out_q_scale4[v2, v3 // 32], T.float16(0.0)) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size and ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1 < vocab_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) T.writes(compute_intermediate_intermediate[v1, 0, v2]) compute_intermediate_intermediate[v1, 0, v2] = T.Cast("float32", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5(p_embed_out_q_weight3: T.handle, p_embed_out_q_scale3: T.handle, p_take1: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() embed_out_q_weight3 = T.match_buffer(p_embed_out_q_weight3, (vocab_size, 64), "uint32") embed_out_q_scale3 = T.match_buffer(p_embed_out_q_scale3, (vocab_size, 16), "float16") batch_size = T.int32() take1 = T.match_buffer(p_take1, (1, batch_size, 512), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, batch_size, vocab_size)) # with T.block("root"): NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, (vocab_size + 255) // 256 * 256), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding((vocab_size + 255) // 256, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(32, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(512, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], take1[0, v1, v3], embed_out_q_weight3[v2, v3 // 8], embed_out_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, take1[0, v1, v3], T.float16(0.0)) * T.if_then_else(v2 < vocab_size, (T.Cast("float16", T.bitwise_and(T.shift_right(embed_out_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * embed_out_q_scale3[v2, v3 // 32], T.float16(0.0)) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size and ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1 < vocab_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) T.writes(compute_intermediate_intermediate[0, v1, v2]) compute_intermediate_intermediate[0, v1, v2] = T.Cast("float32", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_take1(p_gpt_neox_embed_in_q_weight: T.handle, p_gpt_neox_embed_in_q_scale: T.handle, p_input_ids: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() gpt_neox_embed_in_q_weight = T.match_buffer(p_gpt_neox_embed_in_q_weight, (vocab_size, 64), "uint32") gpt_neox_embed_in_q_scale = T.match_buffer(p_gpt_neox_embed_in_q_scale, (vocab_size, 16), "float16") seq_len = T.int32() input_ids = T.match_buffer(p_input_ids, (seq_len,), "int32") T_take_intermediate = T.match_buffer(p_output0, (seq_len, 512), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 2, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_take"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 512) v1 = T.axis.spatial(512, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 512) T.reads(input_ids[v0], gpt_neox_embed_in_q_weight[input_ids[v0], v1 // 8], gpt_neox_embed_in_q_scale[input_ids[v0], v1 // 32]) T.writes(T_take_intermediate[v0, v1]) T_take_intermediate[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_embed_in_q_weight[input_ids[v0], v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_embed_in_q_scale[input_ids[v0], v1 // 32] @T.prim_func def gpt_neox_q4f16_1_fused_reshape10_reshape11(lv34: T.Buffer((1, 8, 64), "float16"), T_reshape_intermediate_1: T.Buffer((1, 1, 512), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_fused_0 in T.thread_binding(2, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape_1"): v0 = T.axis.spatial(512, ax0_fused_0 * 256 + ax0_fused_1) T.reads(lv34[0, v0 // 64, v0 % 64]) T.writes(T_reshape_intermediate_1[0, 0, v0]) T_reshape_intermediate_1[0, 0, v0] = lv34[0, v0 // 64, v0 % 64] @T.prim_func def gpt_neox_q4f16_1_fused_reshape8_reshape9(add36: T.Buffer((1, 1, 1536), "float16"), T_reshape_intermediate_1: T.Buffer((1, 24, 64), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(6, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape_1"): v0 = T.axis.spatial(24, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 64) v1 = T.axis.spatial(64, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 64) T.reads(add36[0, 0, v0 * 64 + v1]) T.writes(T_reshape_intermediate_1[0, v0, v1]) T_reshape_intermediate_1[0, v0, v1] = add36[0, 0, v0 * 64 + v1] @T.prim_func def gpt_neox_q4f16_1_fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() qkv = T.match_buffer(var_qkv, (seq_len, 24, 64), "float16") position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) q = T.match_buffer(var_q, (seq_len, 8, 64), "float16") k = T.match_buffer(var_k, (seq_len, 8, 64), "float16") v = T.match_buffer(var_v, (seq_len, 8, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 6, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("llama_fused_rope"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1536) v1 = T.axis.spatial(24, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1536 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(position_map[v0], qkv[v0, v1, v2 + -8:v2 + -8 + 17]) T.writes(q[v0, v1, v2], k[v0, v1 + -8, v2], v[v0, v1 + -16, v2]) if v1 < 8: freq = T.float32() q[v0, v1, v2] = T.if_then_else(apply_rope > 0 and v2 < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[v0, v1, v2]) + T.sin(freq) * T.Cast("float32", T.if_then_else(v2 < 8, qkv[v0, v1, v2 + 8] * T.float16(-1.0), qkv[v0, v1, v2 + -8]))), where={freq: T.Cast("float32", position_map[v0]) / T.pow(T.float32(10000.0), T.Cast("float32", v2 * 2 % 16) / T.float32(16.0))}), qkv[v0, v1, v2]) else: if v1 < 16: freq = T.float32() k[v0, v1 + -8, v2] = T.if_then_else(apply_rope > 0 and v2 < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[v0, v1, v2]) + T.sin(freq) * T.Cast("float32", T.if_then_else(v2 < 8, qkv[v0, v1, v2 + 8] * T.float16(-1.0), qkv[v0, v1, v2 + -8]))), where={freq: T.Cast("float32", position_map[v0]) / T.pow(T.float32(10000.0), T.Cast("float32", v2 * 2 % 16) / T.float32(16.0))}), qkv[v0, v1, v2]) else: v[v0, v1 + -16, v2] = qkv[v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (m, n)) batch_size = T.int32(is_size_var=True) indices = T.match_buffer(var_indices, (batch_size,), "int32") dst = T.match_buffer(var_dst, (batch_size, n)) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding((batch_size * n + 255) // 256, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("gather_2d"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % (n * batch_size) // n) v1 = T.axis.spatial(n, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % n) T.where(ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1 < batch_size * n) T.reads(src[indices[v0], v1], indices[v0]) T.writes(dst[v0, v1]) dst[v0, v1] = src[indices[v0], v1] @T.prim_func def gpt_neox_q4f16_1_index(var_layer_norm12: T.handle, index: T.Buffer((1, 1, 512), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm12 = T.match_buffer(var_layer_norm12, (1, seq_len, 512), "float16") # with T.block("root"): for ax0_fused_0 in T.thread_binding(2, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("index"): v0 = T.axis.spatial(512, ax0_fused_0 * 256 + ax0_fused_1) T.reads(layer_norm12[0, seq_len - 1, v0]) T.writes(index[0, 0, v0]) index[0, 0, v0] = layer_norm12[0, seq_len - 1, v0] @T.prim_func def gpt_neox_q4f16_1_layer_norm(var_input_embeds: T.handle, gpt_neox_layers_0_input_layernorm_weight4: T.Buffer((512,), "float16"), gpt_neox_layers_0_input_layernorm_bias4: T.Buffer((512,), "float16"), var_T_layer_norm: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() input_embeds = T.match_buffer(var_input_embeds, (batch_size, 1, 512), "float16") T_layer_norm = T.match_buffer(var_T_layer_norm, (batch_size, 1, 512), "float16") # with T.block("root"): input_embeds_red_temp_v0_shared = T.alloc_buffer((batch_size, 1), scope="shared") input_embeds_red_temp_v1_shared = T.alloc_buffer((batch_size, 1), scope="shared") for ax0_fused in T.thread_binding(batch_size, thread="blockIdx.x"): for ax0 in range(1): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("input_embeds_red_temp"): v0 = T.axis.spatial(batch_size, ax0_fused + ax0) v1 = T.axis.reduce(512, ax1_fused_0 * 256 + ax1_fused_1) T.reads(input_embeds[v0, 0, v1]) T.writes(input_embeds_red_temp_v0_shared[v0, 0], input_embeds_red_temp_v1_shared[v0, 0]) with T.init(): input_embeds_red_temp_v0_shared[v0, 0] = T.float32(0.0) input_embeds_red_temp_v1_shared[v0, 0] = T.float32(0.0) v_input_embeds_red_temp_v0: T.float32 = input_embeds_red_temp_v0_shared[v0, 0] + T.Cast("float32", input_embeds[v0, 0, v1]) v_input_embeds_red_temp_v1: T.float32 = input_embeds_red_temp_v1_shared[v0, 0] + T.Cast("float32", input_embeds[v0, 0, v1]) * T.Cast("float32", input_embeds[v0, 0, v1]) input_embeds_red_temp_v0_shared[v0, 0] = v_input_embeds_red_temp_v0 input_embeds_red_temp_v1_shared[v0, 0] = v_input_embeds_red_temp_v1 for ax1_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("T_layer_norm"): v0 = T.axis.spatial(batch_size, ax0_fused) v1 = T.axis.spatial(512, ax1_0 * 256 + ax1_1) T.reads(input_embeds[v0, 0, v1], input_embeds_red_temp_v0_shared[v0, 0], input_embeds_red_temp_v1_shared[v0, 0], gpt_neox_layers_0_input_layernorm_weight4[v1], gpt_neox_layers_0_input_layernorm_bias4[v1]) T.writes(T_layer_norm[v0, 0, v1]) T_layer_norm[v0, 0, v1] = T.Cast("float16", (T.Cast("float32", input_embeds[v0, 0, v1]) - input_embeds_red_temp_v0_shared[v0, 0] * T.float32(0.001953125)) * T.rsqrt(input_embeds_red_temp_v1_shared[v0, 0] * T.float32(0.001953125) - input_embeds_red_temp_v0_shared[v0, 0] * T.float32(0.001953125) * (input_embeds_red_temp_v0_shared[v0, 0] * T.float32(0.001953125)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight4[v1] + gpt_neox_layers_0_input_layernorm_bias4[v1] @T.prim_func def gpt_neox_q4f16_1_layer_norm1(var_input_embeds: T.handle, gpt_neox_layers_0_input_layernorm_weight3: T.Buffer((512,), "float16"), gpt_neox_layers_0_input_layernorm_bias3: T.Buffer((512,), "float16"), var_T_layer_norm: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() input_embeds = T.match_buffer(var_input_embeds, (1, seq_len, 512), "float16") T_layer_norm = T.match_buffer(var_T_layer_norm, (1, seq_len, 512), "float16") # with T.block("root"): input_embeds_red_temp_v0_shared = T.alloc_buffer((1, seq_len), scope="shared") input_embeds_red_temp_v1_shared = T.alloc_buffer((1, seq_len), scope="shared") for ax0_fused in T.thread_binding(seq_len, thread="blockIdx.x"): for ax0 in range(1): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("input_embeds_red_temp"): v0 = T.axis.spatial(seq_len, ax0_fused + ax0) v1 = T.axis.reduce(512, ax1_fused_0 * 256 + ax1_fused_1) T.reads(input_embeds[0, v0, v1]) T.writes(input_embeds_red_temp_v0_shared[0, v0], input_embeds_red_temp_v1_shared[0, v0]) with T.init(): input_embeds_red_temp_v0_shared[0, v0] = T.float32(0.0) input_embeds_red_temp_v1_shared[0, v0] = T.float32(0.0) v_input_embeds_red_temp_v0: T.float32 = input_embeds_red_temp_v0_shared[0, v0] + T.Cast("float32", input_embeds[0, v0, v1]) v_input_embeds_red_temp_v1: T.float32 = input_embeds_red_temp_v1_shared[0, v0] + T.Cast("float32", input_embeds[0, v0, v1]) * T.Cast("float32", input_embeds[0, v0, v1]) input_embeds_red_temp_v0_shared[0, v0] = v_input_embeds_red_temp_v0 input_embeds_red_temp_v1_shared[0, v0] = v_input_embeds_red_temp_v1 for ax1_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("T_layer_norm"): v0 = T.axis.spatial(seq_len, ax0_fused) v1 = T.axis.spatial(512, ax1_0 * 256 + ax1_1) T.reads(input_embeds[0, v0, v1], input_embeds_red_temp_v0_shared[0, v0], input_embeds_red_temp_v1_shared[0, v0], gpt_neox_layers_0_input_layernorm_weight3[v1], gpt_neox_layers_0_input_layernorm_bias3[v1]) T.writes(T_layer_norm[0, v0, v1]) T_layer_norm[0, v0, v1] = T.Cast("float16", (T.Cast("float32", input_embeds[0, v0, v1]) - input_embeds_red_temp_v0_shared[0, v0] * T.float32(0.001953125)) * T.rsqrt(input_embeds_red_temp_v1_shared[0, v0] * T.float32(0.001953125) - input_embeds_red_temp_v0_shared[0, v0] * T.float32(0.001953125) * (input_embeds_red_temp_v0_shared[0, v0] * T.float32(0.001953125)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight3[v1] + gpt_neox_layers_0_input_layernorm_bias3[v1] @T.prim_func def gpt_neox_q4f16_1_layer_norm2(input_embed: T.Buffer((1, 1, 512), "float16"), gpt_neox_layers_0_input_layernorm_weight2: T.Buffer((512,), "float16"), gpt_neox_layers_0_input_layernorm_bias2: T.Buffer((512,), "float16"), T_layer_norm: T.Buffer((1, 1, 512), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): input_embed_red_temp_v0_shared = T.alloc_buffer((1, 1), scope="shared") input_embed_red_temp_v1_shared = T.alloc_buffer((1, 1), scope="shared") for ax0_fused in T.thread_binding(1, thread="blockIdx.x"): for ax0 in range(1): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_fused_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("input_embed_red_temp"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.reduce(512, ax1_fused_0 * 256 + ax1_fused_1) T.reads(input_embed[0, 0, v1]) T.writes(input_embed_red_temp_v0_shared[0, 0], input_embed_red_temp_v1_shared[0, 0]) with T.init(): input_embed_red_temp_v0_shared[0, 0] = T.float32(0.0) input_embed_red_temp_v1_shared[0, 0] = T.float32(0.0) v_input_embed_red_temp_v0: T.float32 = input_embed_red_temp_v0_shared[0, 0] + T.Cast("float32", input_embed[0, 0, v1]) v_input_embed_red_temp_v1: T.float32 = input_embed_red_temp_v1_shared[0, 0] + T.Cast("float32", input_embed[0, 0, v1]) * T.Cast("float32", input_embed[0, 0, v1]) input_embed_red_temp_v0_shared[0, 0] = v_input_embed_red_temp_v0 input_embed_red_temp_v1_shared[0, 0] = v_input_embed_red_temp_v1 for ax1_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_0 in T.serial(2, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("T_layer_norm"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial(512, ax1_0 * 256 + ax1_1) T.reads(input_embed[0, 0, v1], input_embed_red_temp_v0_shared[0, 0], input_embed_red_temp_v1_shared[0, 0], gpt_neox_layers_0_input_layernorm_weight2[v1], gpt_neox_layers_0_input_layernorm_bias2[v1]) T.writes(T_layer_norm[0, 0, v1]) T_layer_norm[0, 0, v1] = T.Cast("float16", (T.Cast("float32", input_embed[0, 0, v1]) - input_embed_red_temp_v0_shared[0, 0] * T.float32(0.001953125)) * T.rsqrt(input_embed_red_temp_v1_shared[0, 0] * T.float32(0.001953125) - input_embed_red_temp_v0_shared[0, 0] * T.float32(0.001953125) * (input_embed_red_temp_v0_shared[0, 0] * T.float32(0.001953125)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight2[v1] + gpt_neox_layers_0_input_layernorm_bias2[v1] @T.prim_func def gpt_neox_q4f16_1_merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) V = T.match_buffer(v, (N, H, D), "float16") S = T.match_buffer(s, (N, H)) V_other = T.match_buffer(v_other, (N, H, D), "float16") S_other = T.match_buffer(s_other, (N, H)) # with T.block("root"): for bx in T.thread_binding(N, thread="blockIdx.x"): for by in T.thread_binding(1, thread="blockIdx.y"): for ty in T.thread_binding(8, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): with T.block("merge"): T.reads(S[bx, ty + by * 8], S_other[bx, ty + by * 8], V[bx, ty + by * 8, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 8, tx * 4:tx * 4 + 4]) T.writes(V[bx, ty + by * 8, tx * 4:tx * 4 + 4], S[bx, ty + by * 8]) s_val = T.alloc_buffer((1,), scope="local") s_other_val = T.alloc_buffer((1,), scope="local") s_max = T.alloc_buffer((1,), scope="local") scale = T.alloc_buffer((1,), scope="local") other_scale = T.alloc_buffer((1,), scope="local") v_vec = T.alloc_buffer((4,), "float16", scope="local") v_other_vec = T.alloc_buffer((4,), "float16", scope="local") s_val[0] = S[bx, ty + by * 8] s_other_val[0] = S_other[bx, ty + by * 8] s_max[0] = T.max(s_val[0], s_other_val[0]) s_val[0] = T.exp2(s_val[0] - s_max[0]) s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) for vec in T.vectorized(4): v_vec[vec] = V[bx, ty + by * 8, tx * 4 + vec] for vec in T.vectorized(4): v_other_vec[vec] = V_other[bx, ty + by * 8, tx * 4 + vec] for vec in range(4): v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) for vec in T.vectorized(4): V[bx, ty + by * 8, tx * 4 + vec] = v_vec[vec] S[bx, ty + by * 8] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] @T.prim_func def gpt_neox_q4f16_1_reshape(var_add108: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() add108 = T.match_buffer(var_add108, (batch_size, 1, 1536), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 24, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 6, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1536) v1 = T.axis.spatial(24, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1536 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(add108[v0, 0, v1 * 64 + v2]) T.writes(T_reshape[v0, 0, v1, v2]) T_reshape[v0, 0, v1, v2] = add108[v0, 0, v1 * 64 + v2] @T.prim_func def gpt_neox_q4f16_1_reshape1(var_reshape72: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape72 = T.match_buffer(var_reshape72, (batch_size, 1, 24, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 24, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 6, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1536) v1 = T.axis.spatial(24, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1536 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(reshape72[v0, 0, v1, v2]) T.writes(T_reshape[v0, v1, v2]) T_reshape[v0, v1, v2] = reshape72[v0, 0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape2(var_lv96: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv96 = T.match_buffer(var_lv96, (batch_size, 8, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 8, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 2, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 512) v1 = T.axis.spatial(8, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 512 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(lv96[v0, v1, v2]) T.writes(T_reshape[v0, 0, v1, v2]) T_reshape[v0, 0, v1, v2] = lv96[v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape3(var_reshape74: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape74 = T.match_buffer(var_reshape74, (batch_size, 1, 8, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 512), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 2, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 512) v1 = T.axis.spatial(512, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 512) T.reads(reshape74[v0, 0, v1 // 64, v1 % 64]) T.writes(T_reshape[v0, 0, v1]) T_reshape[v0, 0, v1] = reshape74[v0, 0, v1 // 64, v1 % 64] @T.prim_func def gpt_neox_q4f16_1_reshape4(var_add72: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() add72 = T.match_buffer(var_add72, (1, seq_len, 1536), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 24, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 6, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1536) v1 = T.axis.spatial(24, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1536 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(add72[0, v0, v1 * 64 + v2]) T.writes(T_reshape[0, v0, v1, v2]) T_reshape[0, v0, v1, v2] = add72[0, v0, v1 * 64 + v2] @T.prim_func def gpt_neox_q4f16_1_reshape5(var_reshape48: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape48 = T.match_buffer(var_reshape48, (1, seq_len, 24, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (seq_len, 24, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 6, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1536) v1 = T.axis.spatial(24, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1536 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(reshape48[0, v0, v1, v2]) T.writes(T_reshape[v0, v1, v2]) T_reshape[v0, v1, v2] = reshape48[0, v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape6(var_lv65: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv65 = T.match_buffer(var_lv65, (seq_len, 8, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 8, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 2, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 512) v1 = T.axis.spatial(8, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 512 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(lv65[v0, v1, v2]) T.writes(T_reshape[0, v0, v1, v2]) T_reshape[0, v0, v1, v2] = lv65[v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape7(var_reshape50: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape50 = T.match_buffer(var_reshape50, (1, seq_len, 8, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 512), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 2, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 512) v1 = T.axis.spatial(512, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 512) T.reads(reshape50[0, v0, v1 // 64, v1 % 64]) T.writes(T_reshape[0, v0, v1]) T_reshape[0, v0, v1] = reshape50[0, v0, v1 // 64, v1 % 64] @T.prim_func def gpt_neox_q4f16_1_scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (batch_size, n)) indices = T.match_buffer(var_indices, (batch_size,), "int32") m = T.int32(is_size_var=True) dst = T.match_buffer(var_dst, (m, n)) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding((batch_size * n + 255) // 256, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("scatter_2d"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % (n * batch_size) // n) v1 = T.axis.spatial(n, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % n) T.where(ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1 < batch_size * n) T.reads(src[v0, v1], indices[v0]) T.writes(dst[indices[v0], v1]) dst[indices[v0], v1] = src[v0, v1] @T.prim_func def gpt_neox_q4f16_1_shape_func(H: T.Buffer((T.int64(5),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(4)] = T.int64(512) * H[T.int64(1)] H[T.int64(3)] = (H[T.int64(1)] + T.int64(4096) - T.int64(1)) // T.int64(4096) H[T.int64(2)] = T.int64(512) * ((H[T.int64(1)] + T.int64(4096) - T.int64(1)) // T.int64(4096)) @T.prim_func def gpt_neox_q4f16_1_shape_func1(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = T.int64(128) * H[T.int64(1)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func2(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = T.int64(2048) * H[T.int64(1)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func3(H: T.Buffer((T.int64(3),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(2)] = H[T.int64(0)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func4(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = H[T.int64(1)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func5(H: T.Buffer((T.int64(5),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(4)] = T.int64(128) * H[T.int64(2)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int32(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0_1 in T.thread_binding(32, thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + 31) // 32, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * 32 + ax0_1) T.where(ax0_0 * 32 + ax0_1 < num_chunks) T.reads(chunked_max[v0, v1]) T.writes(temp_max_shared[v0]) with T.init(): temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) for ax0_1 in T.thread_binding(32, thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + 31) // 32, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * 32 + ax0_1) T.where(ax0_0 * 32 + ax0_1 < num_chunks) T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) T.writes(temp_sum_shared[v0]) with T.init(): temp_sum_shared[v0] = T.float32(0.0) temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) for l2_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): for l2_1 in T.thread_binding(8, thread="threadIdx.y"): for l2_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("log_pad"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) v2 = T.axis.spatial(4096, l2_0 * 256 + l2_1 * 32 + l2_2) T.reads(temperature[v0], A[v0, v1 * 4096 + v2], temp_sum_shared[v0], temp_max_shared[v0]) T.writes(softmax[v0, v1 * 4096 + v2]) if v1 * 4096 + v2 < vocab_size: softmax[v0, v1 * 4096 + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * 4096 + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * 4096 + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) @T.prim_func def gpt_neox_q4f16_1_take(var_layer_norm38: T.handle, var_logit_positions: T.handle, var_T_take: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm38 = T.match_buffer(var_layer_norm38, (1, seq_len, 512), "float16") batch_size = T.int32() logit_positions = T.match_buffer(var_logit_positions, (batch_size,), "int32") T_take = T.match_buffer(var_T_take, (1, batch_size, 512), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 2, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_take"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 512) v1 = T.axis.spatial(512, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 512) T.reads(layer_norm38[0, logit_positions[v0], v1], logit_positions[v0]) T.writes(T_take[0, v0, v1]) T_take[0, v0, v1] = layer_norm38[0, logit_positions[v0], v1] @T.prim_func def gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) num_pages, page_size = T.int32(), T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (num_pages, 2, 8, page_size, 64), "float16", offset_factor=1) seqlen = T.int32(is_size_var=True) position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) k_data = T.match_buffer(var_k_data, (6, seqlen, 8, 64), "float16") v_data = T.match_buffer(var_v_data, (6, seqlen, 8, 64), "float16") # with T.block("root"): for p_h_d_fused_0 in T.thread_binding(seqlen * 2, thread="blockIdx.x"): for p_h_d_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy0"): vp = T.axis.spatial(seqlen, (p_h_d_fused_0 * 256 + p_h_d_fused_1) // 512) vh = T.axis.spatial(8, (p_h_d_fused_0 * 256 + p_h_d_fused_1) % 512 // 64) vd = T.axis.spatial(64, (p_h_d_fused_0 * 256 + p_h_d_fused_1) % 64) T.reads(position_map[vp], pages[position_map[vp] // page_size, 0:2, vh, position_map[vp] % page_size, vd]) T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) position: T.int32 = position_map[vp] k_data[layer_id, vp, vh, vd] = pages[position // page_size, 0, vh, position % page_size, vd] v_data[layer_id, vp, vh, vd] = pages[position // page_size, 1, vh, position % page_size, vd] @T.prim_func def gpt_neox_q4f16_1_tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 8, 16, 64), "float16", offset_factor=1) ntoken = T.int32(is_size_var=True) k_data = T.match_buffer(var_k_data, (ntoken, 8, 64), "float16") v_data = T.match_buffer(var_v_data, (ntoken, 8, 64), "float16") position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) # with T.block("root"): for global_pos_h_f_fused_0 in T.thread_binding(ntoken * 2, thread="blockIdx.x"): for global_pos_h_f_fused_1 in T.thread_binding(256, thread="threadIdx.x"): if position_map[(global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 512] != -1: with T.block("k_transpose_append"): vgpos = T.axis.spatial(ntoken, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 512) vh = T.axis.spatial(8, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 512 // 64) vf = T.axis.spatial(64, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 64) T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] with T.block("v_transpose_append"): vgpos = T.axis.spatial(ntoken, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 512) vh = T.axis.spatial(8, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 512 // 64) vf = T.axis.spatial(64, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 64) T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] @T.prim_func def gpt_neox_q4f16_1_tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 32, 64), "float16") page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) total_tree_order_len = T.int32(is_size_var=True) tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) # with T.block("root"): assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((32, 64), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 K_smem[i, j] = pages[page_no, 0, by, page_offset, j] else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @R.function def _metadata() -> R.Object: shape_heap: R.Object = R.null_value() return R.str("{\"model_type\": \"gpt_neox\", \"quantization\": \"q4f16_1\", \"context_window_size\": 2048, \"sliding_window_size\": -1, \"attention_sink_size\": -1, \"prefill_chunk_size\": 2048, \"tensor_parallel_shards\": 1, \"pipeline_parallel_stages\": 1, \"disaggregation\": false, \"kv_state_kind\": \"kv_cache\", \"max_batch_size\": 128, \"params\": [{\"name\": \"gpt_neox.embed_in.q_weight\", \"shape\": [\"vocab_size\", 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.embed_in.q_scale\", \"shape\": [\"vocab_size\", 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.input_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.input_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.post_attention_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.post_attention_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.query_key_value.q_weight\", \"shape\": [1536, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.query_key_value.q_scale\", \"shape\": [1536, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.query_key_value.bias\", \"shape\": [1536], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.dense.q_weight\", \"shape\": [512, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.dense.q_scale\", \"shape\": [512, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.dense.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight\", \"shape\": [2048, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale\", \"shape\": [2048, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_h_to_4h.bias\", \"shape\": [2048], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight\", \"shape\": [512, 256], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale\", \"shape\": [512, 64], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_4h_to_h.bias\", \"shape\": [512], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.input_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.input_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.post_attention_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.post_attention_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.query_key_value.q_weight\", \"shape\": [1536, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.query_key_value.q_scale\", \"shape\": [1536, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.query_key_value.bias\", \"shape\": [1536], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.dense.q_weight\", \"shape\": [512, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.dense.q_scale\", \"shape\": [512, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.dense.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight\", \"shape\": [2048, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale\", \"shape\": [2048, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_h_to_4h.bias\", \"shape\": [2048], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight\", \"shape\": [512, 256], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale\", \"shape\": [512, 64], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_4h_to_h.bias\", \"shape\": [512], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.input_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.input_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.post_attention_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.post_attention_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.query_key_value.q_weight\", \"shape\": [1536, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.query_key_value.q_scale\", \"shape\": [1536, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.query_key_value.bias\", \"shape\": [1536], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.dense.q_weight\", \"shape\": [512, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.dense.q_scale\", \"shape\": [512, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.dense.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight\", \"shape\": [2048, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale\", \"shape\": [2048, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_h_to_4h.bias\", \"shape\": [2048], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight\", \"shape\": [512, 256], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale\", \"shape\": [512, 64], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_4h_to_h.bias\", \"shape\": [512], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.input_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.input_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.post_attention_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.post_attention_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.query_key_value.q_weight\", \"shape\": [1536, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.query_key_value.q_scale\", \"shape\": [1536, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.query_key_value.bias\", \"shape\": [1536], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.dense.q_weight\", \"shape\": [512, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.dense.q_scale\", \"shape\": [512, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.dense.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight\", \"shape\": [2048, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale\", \"shape\": [2048, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_h_to_4h.bias\", \"shape\": [2048], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight\", \"shape\": [512, 256], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale\", \"shape\": [512, 64], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_4h_to_h.bias\", \"shape\": [512], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.input_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.input_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.post_attention_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.post_attention_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.query_key_value.q_weight\", \"shape\": [1536, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.query_key_value.q_scale\", \"shape\": [1536, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.query_key_value.bias\", \"shape\": [1536], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.dense.q_weight\", \"shape\": [512, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.dense.q_scale\", \"shape\": [512, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.dense.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight\", \"shape\": [2048, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale\", \"shape\": [2048, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_h_to_4h.bias\", \"shape\": [2048], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight\", \"shape\": [512, 256], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale\", \"shape\": [512, 64], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_4h_to_h.bias\", \"shape\": [512], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.input_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.input_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.post_attention_layernorm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.post_attention_layernorm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.query_key_value.q_weight\", \"shape\": [1536, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.query_key_value.q_scale\", \"shape\": [1536, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.query_key_value.bias\", \"shape\": [1536], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.dense.q_weight\", \"shape\": [512, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.dense.q_scale\", \"shape\": [512, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.dense.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight\", \"shape\": [2048, 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale\", \"shape\": [2048, 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_h_to_4h.bias\", \"shape\": [2048], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight\", \"shape\": [512, 256], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale\", \"shape\": [512, 64], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_4h_to_h.bias\", \"shape\": [512], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.final_layer_norm.weight\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.final_layer_norm.bias\", \"shape\": [512], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"embed_out.q_weight\", \"shape\": [\"vocab_size\", 64], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"embed_out.q_scale\", \"shape\": [\"vocab_size\", 16], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}], \"kv_cache\": {\"num_hidden_layers\": 6, \"num_attention_heads\": 8, \"num_key_value_heads\": 8, \"head_dim\": 64}, \"memory_usage\": {\"alloc_embedding_tensor\": 2097152, \"batch_decode\": 1572864, \"batch_prefill\": 25296896, \"batch_verify\": 25165824, \"create_tir_paged_kv_cache\": 0, \"decode\": 12288, \"embed\": 2097152, \"prefill\": 25166848, \"softmax_with_temperature\": 0}}") @R.function def alloc_embedding_tensor() -> R.Tensor((2048, 512), dtype="float16"): R.func_attr({"relax.force_pure": True}) shape_heap: R.Object = R.null_value() storage: R.Object = R.vm.alloc_storage(R.shape([2097152]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv: R.Tensor((2048, 512), dtype="float16") = R.vm.alloc_tensor(storage, R.prim_value(0), R.shape([2048, 512]), R.dtype("float16")) R.vm.kill_object(storage) return gv @R.function def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[0], param=input_embeds, annotation=R.Tensor((batch_size, 1, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(102), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv8: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv8, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv9: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv9, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv10: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] R.call_packed("vm.builtin.check_tensor_info", gv10, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv11: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] R.call_packed("vm.builtin.check_tensor_info", gv11, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), R.str("ErrorContext(fn=batch_decode, loc=param[0], param=input_embeds, annotation=R.Tensor((batch_size, 1, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv8, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv9, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv10, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv11, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func1(shape_heap) gpt_neox_layers_0_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[3] storage3: R.Object = R.vm.alloc_storage(R.shape([131072]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv12: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc3: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv12, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight4, gpt_neox_layers_0_input_layernorm_bias4, alloc3) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias4) gpt_neox_layers_0_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[8] storage4: R.Object = R.vm.alloc_storage(R.shape([393216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv13: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc4: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv13, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_0_attention_query_key_value_q_weight4, gpt_neox_layers_0_attention_query_key_value_q_scale4, alloc3, gpt_neox_layers_0_attention_query_key_value_bias4, alloc4) R.vm.kill_object(alloc3) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias4) gv14: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape72: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc4, gv14, sinfo_args=(R.Tensor((batch_size, 1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc4) gv15: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape73: R.Tensor((batch_size, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape72, gv15, sinfo_args=(R.Tensor((batch_size, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape72) gv16: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc5: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv16, R.dtype("float16")) _4: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape73, alloc5) R.vm.kill_object(reshape73) gv17: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape74: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc5, gv17, sinfo_args=(R.Tensor((batch_size, 1, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc5) gv18: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape75: R.Tensor((batch_size, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape74, gv18, sinfo_args=(R.Tensor((batch_size, 1, 512), dtype="float16"),)) R.vm.kill_object(reshape74) gpt_neox_layers_0_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[5] gv19: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc6: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv19, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight4, gpt_neox_layers_0_post_attention_layernorm_bias4, alloc6) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias4) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[14] storage5: R.Object = R.vm.alloc_storage(R.shape([524288]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv20: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc7: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv20, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, alloc6, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, alloc7) R.vm.kill_object(alloc6) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias4) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[17] gv21: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc8: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv21, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, alloc7, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, alloc8) R.vm.kill_object(alloc7) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias4) gpt_neox_layers_0_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[11] gv22: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc9: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv22, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_0_attention_dense_q_weight4, gpt_neox_layers_0_attention_dense_q_scale4, reshape75, gpt_neox_layers_0_attention_dense_bias4, alloc8, input_embeds, alloc9) R.vm.kill_object(reshape75) R.vm.kill_object(alloc8) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias4) gpt_neox_layers_1_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[19] gv23: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc10: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv23, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc9, gpt_neox_layers_1_input_layernorm_weight4, gpt_neox_layers_1_input_layernorm_bias4, alloc10) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias4) gpt_neox_layers_1_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[24] gv24: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc11: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv24, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_1_attention_query_key_value_q_weight4, gpt_neox_layers_1_attention_query_key_value_q_scale4, alloc10, gpt_neox_layers_1_attention_query_key_value_bias4, alloc11) R.vm.kill_object(alloc10) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias4) gv25: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape76: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc11, gv25, sinfo_args=(R.Tensor((batch_size, 1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc11) gv26: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape77: R.Tensor((batch_size, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape76, gv26, sinfo_args=(R.Tensor((batch_size, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape76) gv27: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc12: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv27, R.dtype("float16")) _11: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape77, alloc12) R.vm.kill_object(reshape77) gv28: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape78: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc12, gv28, sinfo_args=(R.Tensor((batch_size, 1, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc12) gv29: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape79: R.Tensor((batch_size, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape78, gv29, sinfo_args=(R.Tensor((batch_size, 1, 512), dtype="float16"),)) R.vm.kill_object(reshape78) gpt_neox_layers_1_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[21] gv30: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc13: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv30, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc9, gpt_neox_layers_1_post_attention_layernorm_weight4, gpt_neox_layers_1_post_attention_layernorm_bias4, alloc13) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias4) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[30] storage6: R.Object = R.vm.alloc_storage(R.shape([524288]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv31: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc14: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv31, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4, alloc13, gpt_neox_layers_1_mlp_dense_h_to_4h_bias4, alloc14) R.vm.kill_object(alloc13) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias4) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[33] gv32: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc15: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv32, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4, alloc14, gpt_neox_layers_1_mlp_dense_4h_to_h_bias4, alloc15) R.vm.kill_object(alloc14) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias4) gpt_neox_layers_1_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[27] gv33: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc16: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv33, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_1_attention_dense_q_weight4, gpt_neox_layers_1_attention_dense_q_scale4, reshape79, gpt_neox_layers_1_attention_dense_bias4, alloc15, alloc9, alloc16) R.vm.kill_object(alloc9) R.vm.kill_object(reshape79) R.vm.kill_object(alloc15) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias4) gpt_neox_layers_2_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[35] gv34: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc17: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv34, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc16, gpt_neox_layers_2_input_layernorm_weight4, gpt_neox_layers_2_input_layernorm_bias4, alloc17) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias4) gpt_neox_layers_2_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[40] gv35: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc18: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv35, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_2_attention_query_key_value_q_weight4, gpt_neox_layers_2_attention_query_key_value_q_scale4, alloc17, gpt_neox_layers_2_attention_query_key_value_bias4, alloc18) R.vm.kill_object(alloc17) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias4) gv36: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape80: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc18, gv36, sinfo_args=(R.Tensor((batch_size, 1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc18) gv37: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape81: R.Tensor((batch_size, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape80, gv37, sinfo_args=(R.Tensor((batch_size, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape80) gv38: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc19: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv38, R.dtype("float16")) _18: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape81, alloc19) R.vm.kill_object(reshape81) gv39: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape82: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc19, gv39, sinfo_args=(R.Tensor((batch_size, 1, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc19) gv40: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape83: R.Tensor((batch_size, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape82, gv40, sinfo_args=(R.Tensor((batch_size, 1, 512), dtype="float16"),)) R.vm.kill_object(reshape82) gpt_neox_layers_2_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[37] gv41: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc20: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv41, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc16, gpt_neox_layers_2_post_attention_layernorm_weight4, gpt_neox_layers_2_post_attention_layernorm_bias4, alloc20) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias4) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[46] gv42: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc21: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv42, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4, alloc20, gpt_neox_layers_2_mlp_dense_h_to_4h_bias4, alloc21) R.vm.kill_object(alloc20) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias4) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[49] gv43: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc22: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv43, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4, alloc21, gpt_neox_layers_2_mlp_dense_4h_to_h_bias4, alloc22) R.vm.kill_object(alloc21) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias4) gpt_neox_layers_2_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[43] gv44: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc23: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv44, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_2_attention_dense_q_weight4, gpt_neox_layers_2_attention_dense_q_scale4, reshape83, gpt_neox_layers_2_attention_dense_bias4, alloc22, alloc16, alloc23) R.vm.kill_object(alloc16) R.vm.kill_object(reshape83) R.vm.kill_object(alloc22) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias4) gpt_neox_layers_3_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[51] gv45: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc24: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv45, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc23, gpt_neox_layers_3_input_layernorm_weight4, gpt_neox_layers_3_input_layernorm_bias4, alloc24) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias4) gpt_neox_layers_3_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[56] gv46: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc25: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv46, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_3_attention_query_key_value_q_weight4, gpt_neox_layers_3_attention_query_key_value_q_scale4, alloc24, gpt_neox_layers_3_attention_query_key_value_bias4, alloc25) R.vm.kill_object(alloc24) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias4) gv47: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape84: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc25, gv47, sinfo_args=(R.Tensor((batch_size, 1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc25) gv48: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape85: R.Tensor((batch_size, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape84, gv48, sinfo_args=(R.Tensor((batch_size, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape84) gv49: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc26: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv49, R.dtype("float16")) _25: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape85, alloc26) R.vm.kill_object(reshape85) gv50: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape86: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc26, gv50, sinfo_args=(R.Tensor((batch_size, 1, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc26) gv51: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape87: R.Tensor((batch_size, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape86, gv51, sinfo_args=(R.Tensor((batch_size, 1, 512), dtype="float16"),)) R.vm.kill_object(reshape86) gpt_neox_layers_3_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[53] gv52: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc27: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv52, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc23, gpt_neox_layers_3_post_attention_layernorm_weight4, gpt_neox_layers_3_post_attention_layernorm_bias4, alloc27) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias4) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[62] gv53: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc28: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv53, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4, alloc27, gpt_neox_layers_3_mlp_dense_h_to_4h_bias4, alloc28) R.vm.kill_object(alloc27) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias4) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[65] gv54: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc29: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv54, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4, alloc28, gpt_neox_layers_3_mlp_dense_4h_to_h_bias4, alloc29) R.vm.kill_object(alloc28) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias4) gpt_neox_layers_3_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[59] gv55: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc30: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv55, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_3_attention_dense_q_weight4, gpt_neox_layers_3_attention_dense_q_scale4, reshape87, gpt_neox_layers_3_attention_dense_bias4, alloc29, alloc23, alloc30) R.vm.kill_object(alloc23) R.vm.kill_object(reshape87) R.vm.kill_object(alloc29) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias4) gpt_neox_layers_4_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[67] gv56: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc31: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv56, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc30, gpt_neox_layers_4_input_layernorm_weight4, gpt_neox_layers_4_input_layernorm_bias4, alloc31) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias4) gpt_neox_layers_4_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[72] gv57: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc32: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv57, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_4_attention_query_key_value_q_weight4, gpt_neox_layers_4_attention_query_key_value_q_scale4, alloc31, gpt_neox_layers_4_attention_query_key_value_bias4, alloc32) R.vm.kill_object(alloc31) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias4) gv58: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape88: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc32, gv58, sinfo_args=(R.Tensor((batch_size, 1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc32) gv59: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape89: R.Tensor((batch_size, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape88, gv59, sinfo_args=(R.Tensor((batch_size, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape88) gv60: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc33: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv60, R.dtype("float16")) _32: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape89, alloc33) R.vm.kill_object(reshape89) gv61: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape90: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc33, gv61, sinfo_args=(R.Tensor((batch_size, 1, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc33) gv62: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape91: R.Tensor((batch_size, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape90, gv62, sinfo_args=(R.Tensor((batch_size, 1, 512), dtype="float16"),)) R.vm.kill_object(reshape90) gpt_neox_layers_4_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[69] gv63: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc34: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv63, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc30, gpt_neox_layers_4_post_attention_layernorm_weight4, gpt_neox_layers_4_post_attention_layernorm_bias4, alloc34) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias4) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[78] gv64: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc35: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv64, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4, alloc34, gpt_neox_layers_4_mlp_dense_h_to_4h_bias4, alloc35) R.vm.kill_object(alloc34) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias4) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[81] gv65: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc36: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv65, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4, alloc35, gpt_neox_layers_4_mlp_dense_4h_to_h_bias4, alloc36) R.vm.kill_object(alloc35) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias4) gpt_neox_layers_4_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[75] gv66: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc37: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv66, R.dtype("float16")) R.vm.kill_object(storage5) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_4_attention_dense_q_weight4, gpt_neox_layers_4_attention_dense_q_scale4, reshape91, gpt_neox_layers_4_attention_dense_bias4, alloc36, alloc30, alloc37) R.vm.kill_object(alloc30) R.vm.kill_object(reshape91) R.vm.kill_object(alloc36) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias4) gpt_neox_layers_5_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[83] gv67: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc38: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv67, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc37, gpt_neox_layers_5_input_layernorm_weight4, gpt_neox_layers_5_input_layernorm_bias4, alloc38) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias4) gpt_neox_layers_5_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[88] gv68: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc39: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv68, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_5_attention_query_key_value_q_weight4, gpt_neox_layers_5_attention_query_key_value_q_scale4, alloc38, gpt_neox_layers_5_attention_query_key_value_bias4, alloc39) R.vm.kill_object(alloc38) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias4) gv69: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape92: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc39, gv69, sinfo_args=(R.Tensor((batch_size, 1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc39) gv70: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape93: R.Tensor((batch_size, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape92, gv70, sinfo_args=(R.Tensor((batch_size, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape92) gv71: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc40: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv71, R.dtype("float16")) _39: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape93, alloc40) R.vm.kill_object(reshape93) gv72: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape94: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc40, gv72, sinfo_args=(R.Tensor((batch_size, 1, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc40) gv73: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape95: R.Tensor((batch_size, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape94, gv73, sinfo_args=(R.Tensor((batch_size, 1, 512), dtype="float16"),)) R.vm.kill_object(reshape94) gpt_neox_layers_5_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[85] gv74: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc41: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv74, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc37, gpt_neox_layers_5_post_attention_layernorm_weight4, gpt_neox_layers_5_post_attention_layernorm_bias4, alloc41) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias4) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[94] gv75: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc42: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv75, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4, alloc41, gpt_neox_layers_5_mlp_dense_h_to_4h_bias4, alloc42) R.vm.kill_object(alloc41) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias4) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[97] gv76: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc43: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv76, R.dtype("float16")) R.vm.kill_object(storage4) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4, alloc42, gpt_neox_layers_5_mlp_dense_4h_to_h_bias4, alloc43) R.vm.kill_object(alloc42) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias4) gpt_neox_layers_5_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[91] gv77: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc44: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv77, R.dtype("float16")) R.vm.kill_object(storage6) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_5_attention_dense_q_weight4, gpt_neox_layers_5_attention_dense_q_scale4, reshape95, gpt_neox_layers_5_attention_dense_bias4, alloc43, alloc37, alloc44) R.vm.kill_object(alloc37) R.vm.kill_object(reshape95) R.vm.kill_object(alloc43) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias4) gpt_neox_final_layer_norm_weight4: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias4: R.Tensor((512,), dtype="float16") = packed_params[99] gv78: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc45: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv78, R.dtype("float16")) R.vm.kill_object(storage3) cls.gpt_neox_q4f16_1_layer_norm(alloc44, gpt_neox_final_layer_norm_weight4, gpt_neox_final_layer_norm_bias4, alloc45) R.vm.kill_object(alloc44) R.vm.kill_object(gpt_neox_final_layer_norm_weight4) R.vm.kill_object(gpt_neox_final_layer_norm_bias4) embed_out_q_weight4: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale4: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] gv79: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage7: R.Object = R.vm.alloc_storage(gv79, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv80: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc46: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv80, R.dtype("float32")) R.vm.kill_object(storage7) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2(embed_out_q_weight4, embed_out_q_scale4, alloc45, alloc46) R.vm.kill_object(alloc45) R.vm.kill_object(embed_out_q_weight4) R.vm.kill_object(embed_out_q_scale4) gv4: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc46, paged_kv_cache R.vm.kill_object(alloc46) gv81: R.Tensor(dtype="float32", ndim=3) = gv4[0] R.call_packed("vm.builtin.match_shape", gv81, shape_heap, R.prim_value(3), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=batch_decode, loc=return, annotation=R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv4 @R.function def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 512), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tensor_info", logit_positions, R.prim_value(1), R.dtype("int32"), R.str("ErrorContext(fn=batch_prefill, loc=param[1], param=logit_positions, annotation=R.Tensor((batch_size,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(102), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv243: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv243, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv244: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv244, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv245: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] R.call_packed("vm.builtin.check_tensor_info", gv245, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv246: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] R.call_packed("vm.builtin.check_tensor_info", gv246, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), R.str("ErrorContext(fn=batch_prefill, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", logit_positions, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=batch_prefill, loc=param[1], param=logit_positions, annotation=R.Tensor((batch_size,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv243, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(2), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv244, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(2), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv245, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(2), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv246, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(2), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func5(shape_heap) gpt_neox_layers_0_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[3] storage25: R.Object = R.vm.alloc_storage(R.shape([2097152]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv247: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc181: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv247, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_input_layernorm_weight3, gpt_neox_layers_0_input_layernorm_bias3, alloc181) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias3) gpt_neox_layers_0_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[8] storage26: R.Object = R.vm.alloc_storage(R.shape([6291456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv248: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc182: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv248, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight3, gpt_neox_layers_0_attention_query_key_value_q_scale3, alloc181, gpt_neox_layers_0_attention_query_key_value_bias3, alloc182) R.vm.kill_object(alloc181) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias3) gv249: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape48: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc182, gv249, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc182) gv250: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape49: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape48, gv250, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape48) gv251: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc183: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv251, R.dtype("float16")) _182: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape49, alloc183) R.vm.kill_object(reshape49) gv252: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape50: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc183, gv252, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc183) gv253: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape51: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape50, gv253, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape50) gpt_neox_layers_0_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[5] gv254: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc184: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv254, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight3, gpt_neox_layers_0_post_attention_layernorm_bias3, alloc184) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias3) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[14] storage27: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv255: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc185: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv255, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, alloc184, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, alloc185) R.vm.kill_object(alloc184) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias3) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[17] gv256: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc186: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv256, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, alloc185, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, alloc186) R.vm.kill_object(alloc185) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias3) gpt_neox_layers_0_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[11] gv257: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc187: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv257, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight3, gpt_neox_layers_0_attention_dense_q_scale3, reshape51, gpt_neox_layers_0_attention_dense_bias3, alloc186, input_embeds, alloc187) R.vm.kill_object(reshape51) R.vm.kill_object(alloc186) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias3) gpt_neox_layers_1_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[19] gv258: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc188: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv258, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc187, gpt_neox_layers_1_input_layernorm_weight3, gpt_neox_layers_1_input_layernorm_bias3, alloc188) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias3) gpt_neox_layers_1_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[24] gv259: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc189: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv259, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_1_attention_query_key_value_q_weight3, gpt_neox_layers_1_attention_query_key_value_q_scale3, alloc188, gpt_neox_layers_1_attention_query_key_value_bias3, alloc189) R.vm.kill_object(alloc188) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias3) gv260: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape52: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc189, gv260, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc189) gv261: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape53: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape52, gv261, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape52) gv262: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc190: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv262, R.dtype("float16")) _189: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape53, alloc190) R.vm.kill_object(reshape53) gv263: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape54: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc190, gv263, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc190) gv264: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape55: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape54, gv264, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape54) gpt_neox_layers_1_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[21] gv265: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc191: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv265, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc187, gpt_neox_layers_1_post_attention_layernorm_weight3, gpt_neox_layers_1_post_attention_layernorm_bias3, alloc191) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias3) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[30] storage28: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv266: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc192: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv266, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3, alloc191, gpt_neox_layers_1_mlp_dense_h_to_4h_bias3, alloc192) R.vm.kill_object(alloc191) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias3) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[33] gv267: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc193: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv267, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3, alloc192, gpt_neox_layers_1_mlp_dense_4h_to_h_bias3, alloc193) R.vm.kill_object(alloc192) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias3) gpt_neox_layers_1_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[27] gv268: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc194: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv268, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_1_attention_dense_q_weight3, gpt_neox_layers_1_attention_dense_q_scale3, reshape55, gpt_neox_layers_1_attention_dense_bias3, alloc193, alloc187, alloc194) R.vm.kill_object(alloc187) R.vm.kill_object(reshape55) R.vm.kill_object(alloc193) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias3) gpt_neox_layers_2_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[35] gv269: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc195: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv269, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc194, gpt_neox_layers_2_input_layernorm_weight3, gpt_neox_layers_2_input_layernorm_bias3, alloc195) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias3) gpt_neox_layers_2_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[40] gv270: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc196: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv270, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_2_attention_query_key_value_q_weight3, gpt_neox_layers_2_attention_query_key_value_q_scale3, alloc195, gpt_neox_layers_2_attention_query_key_value_bias3, alloc196) R.vm.kill_object(alloc195) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias3) gv271: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape56: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc196, gv271, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc196) gv272: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape57: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape56, gv272, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape56) gv273: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc197: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv273, R.dtype("float16")) _196: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape57, alloc197) R.vm.kill_object(reshape57) gv274: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape58: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc197, gv274, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc197) gv275: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape59: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape58, gv275, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape58) gpt_neox_layers_2_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[37] gv276: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc198: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv276, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc194, gpt_neox_layers_2_post_attention_layernorm_weight3, gpt_neox_layers_2_post_attention_layernorm_bias3, alloc198) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias3) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[46] gv277: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc199: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv277, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3, alloc198, gpt_neox_layers_2_mlp_dense_h_to_4h_bias3, alloc199) R.vm.kill_object(alloc198) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias3) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[49] gv278: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc200: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv278, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3, alloc199, gpt_neox_layers_2_mlp_dense_4h_to_h_bias3, alloc200) R.vm.kill_object(alloc199) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias3) gpt_neox_layers_2_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[43] gv279: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc201: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv279, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_2_attention_dense_q_weight3, gpt_neox_layers_2_attention_dense_q_scale3, reshape59, gpt_neox_layers_2_attention_dense_bias3, alloc200, alloc194, alloc201) R.vm.kill_object(alloc194) R.vm.kill_object(reshape59) R.vm.kill_object(alloc200) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias3) gpt_neox_layers_3_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[51] gv280: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc202: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv280, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc201, gpt_neox_layers_3_input_layernorm_weight3, gpt_neox_layers_3_input_layernorm_bias3, alloc202) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias3) gpt_neox_layers_3_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[56] gv281: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc203: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv281, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_3_attention_query_key_value_q_weight3, gpt_neox_layers_3_attention_query_key_value_q_scale3, alloc202, gpt_neox_layers_3_attention_query_key_value_bias3, alloc203) R.vm.kill_object(alloc202) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias3) gv282: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape60: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc203, gv282, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc203) gv283: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape61: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape60, gv283, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape60) gv284: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc204: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv284, R.dtype("float16")) _203: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape61, alloc204) R.vm.kill_object(reshape61) gv285: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape62: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc204, gv285, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc204) gv286: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape63: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape62, gv286, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape62) gpt_neox_layers_3_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[53] gv287: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc205: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv287, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc201, gpt_neox_layers_3_post_attention_layernorm_weight3, gpt_neox_layers_3_post_attention_layernorm_bias3, alloc205) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias3) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[62] gv288: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc206: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv288, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3, alloc205, gpt_neox_layers_3_mlp_dense_h_to_4h_bias3, alloc206) R.vm.kill_object(alloc205) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias3) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[65] gv289: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc207: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv289, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3, alloc206, gpt_neox_layers_3_mlp_dense_4h_to_h_bias3, alloc207) R.vm.kill_object(alloc206) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias3) gpt_neox_layers_3_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[59] gv290: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc208: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv290, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_3_attention_dense_q_weight3, gpt_neox_layers_3_attention_dense_q_scale3, reshape63, gpt_neox_layers_3_attention_dense_bias3, alloc207, alloc201, alloc208) R.vm.kill_object(alloc201) R.vm.kill_object(reshape63) R.vm.kill_object(alloc207) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias3) gpt_neox_layers_4_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[67] gv291: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc209: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv291, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc208, gpt_neox_layers_4_input_layernorm_weight3, gpt_neox_layers_4_input_layernorm_bias3, alloc209) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias3) gpt_neox_layers_4_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[72] gv292: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc210: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv292, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_4_attention_query_key_value_q_weight3, gpt_neox_layers_4_attention_query_key_value_q_scale3, alloc209, gpt_neox_layers_4_attention_query_key_value_bias3, alloc210) R.vm.kill_object(alloc209) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias3) gv293: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape64: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc210, gv293, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc210) gv294: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape65: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape64, gv294, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape64) gv295: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc211: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv295, R.dtype("float16")) _210: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape65, alloc211) R.vm.kill_object(reshape65) gv296: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape66: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc211, gv296, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc211) gv297: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape67: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape66, gv297, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape66) gpt_neox_layers_4_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[69] gv298: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc212: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv298, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc208, gpt_neox_layers_4_post_attention_layernorm_weight3, gpt_neox_layers_4_post_attention_layernorm_bias3, alloc212) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias3) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[78] gv299: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc213: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv299, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3, alloc212, gpt_neox_layers_4_mlp_dense_h_to_4h_bias3, alloc213) R.vm.kill_object(alloc212) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias3) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[81] gv300: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc214: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv300, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3, alloc213, gpt_neox_layers_4_mlp_dense_4h_to_h_bias3, alloc214) R.vm.kill_object(alloc213) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias3) gpt_neox_layers_4_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[75] gv301: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc215: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv301, R.dtype("float16")) R.vm.kill_object(storage27) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_4_attention_dense_q_weight3, gpt_neox_layers_4_attention_dense_q_scale3, reshape67, gpt_neox_layers_4_attention_dense_bias3, alloc214, alloc208, alloc215) R.vm.kill_object(alloc208) R.vm.kill_object(reshape67) R.vm.kill_object(alloc214) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias3) gpt_neox_layers_5_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[83] gv302: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc216: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv302, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc215, gpt_neox_layers_5_input_layernorm_weight3, gpt_neox_layers_5_input_layernorm_bias3, alloc216) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias3) gpt_neox_layers_5_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[88] gv303: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc217: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv303, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_5_attention_query_key_value_q_weight3, gpt_neox_layers_5_attention_query_key_value_q_scale3, alloc216, gpt_neox_layers_5_attention_query_key_value_bias3, alloc217) R.vm.kill_object(alloc216) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias3) gv304: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape68: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc217, gv304, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc217) gv305: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape69: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape68, gv305, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape68) gv306: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc218: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv306, R.dtype("float16")) _217: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape69, alloc218) R.vm.kill_object(reshape69) gv307: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape70: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc218, gv307, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc218) gv308: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape71: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape70, gv308, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape70) gpt_neox_layers_5_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[85] gv309: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc219: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv309, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc215, gpt_neox_layers_5_post_attention_layernorm_weight3, gpt_neox_layers_5_post_attention_layernorm_bias3, alloc219) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias3) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[94] gv310: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc220: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv310, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3, alloc219, gpt_neox_layers_5_mlp_dense_h_to_4h_bias3, alloc220) R.vm.kill_object(alloc219) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias3) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[97] gv311: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc221: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv311, R.dtype("float16")) R.vm.kill_object(storage26) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3, alloc220, gpt_neox_layers_5_mlp_dense_4h_to_h_bias3, alloc221) R.vm.kill_object(alloc220) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias3) gpt_neox_layers_5_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[91] gv312: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc222: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv312, R.dtype("float16")) R.vm.kill_object(storage28) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_5_attention_dense_q_weight3, gpt_neox_layers_5_attention_dense_q_scale3, reshape71, gpt_neox_layers_5_attention_dense_bias3, alloc221, alloc215, alloc222) R.vm.kill_object(alloc215) R.vm.kill_object(reshape71) R.vm.kill_object(alloc221) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias3) gpt_neox_final_layer_norm_weight3: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias3: R.Tensor((512,), dtype="float16") = packed_params[99] gv313: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc223: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv313, R.dtype("float16")) R.vm.kill_object(storage25) cls.gpt_neox_q4f16_1_layer_norm1(alloc222, gpt_neox_final_layer_norm_weight3, gpt_neox_final_layer_norm_bias3, alloc223) R.vm.kill_object(alloc222) R.vm.kill_object(gpt_neox_final_layer_norm_weight3) R.vm.kill_object(gpt_neox_final_layer_norm_bias3) storage29: R.Object = R.vm.alloc_storage(R.shape([131072]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv314: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc224: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv314, R.dtype("float16")) R.vm.kill_object(storage29) cls.gpt_neox_q4f16_1_take(alloc223, logit_positions, alloc224) R.vm.kill_object(alloc223) embed_out_q_weight3: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale3: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] gv315: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=1),)) storage30: R.Object = R.vm.alloc_storage(gv315, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv316: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=3),)) alloc225: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage30, R.prim_value(0), gv316, R.dtype("float32")) R.vm.kill_object(storage30) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5(embed_out_q_weight3, embed_out_q_scale3, alloc224, alloc225) R.vm.kill_object(alloc224) R.vm.kill_object(embed_out_q_weight3) R.vm.kill_object(embed_out_q_scale3) gv3: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc225, paged_kv_cache R.vm.kill_object(alloc225) gv317: R.Tensor(dtype="float32", ndim=3) = gv3[0] R.call_packed("vm.builtin.match_shape", gv317, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.prim_value(3), R.prim_value(2), R.str("ErrorContext(fn=batch_prefill, loc=return, annotation=R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv3 @R.function def batch_verify(input_embeds: R.Tensor((1, "seq_len", 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(102), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv82: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv82, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv83: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv83, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv84: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] R.call_packed("vm.builtin.check_tensor_info", gv84, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv85: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] R.call_packed("vm.builtin.check_tensor_info", gv85, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), R.str("ErrorContext(fn=batch_verify, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv82, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv83, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv84, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv85, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func2(shape_heap) gpt_neox_layers_0_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[3] storage8: R.Object = R.vm.alloc_storage(R.shape([2097152]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv86: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc47: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv86, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_input_layernorm_weight5, gpt_neox_layers_0_input_layernorm_bias5, alloc47) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias5) gpt_neox_layers_0_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[8] storage9: R.Object = R.vm.alloc_storage(R.shape([6291456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv87: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc48: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv87, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight5, gpt_neox_layers_0_attention_query_key_value_q_scale5, alloc47, gpt_neox_layers_0_attention_query_key_value_bias5, alloc48) R.vm.kill_object(alloc47) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias5) gv88: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape96: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc48, gv88, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc48) gv89: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape97: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape96, gv89, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape96) gv90: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc49: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv90, R.dtype("float16")) _48: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape97, alloc49) R.vm.kill_object(reshape97) gv91: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape98: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc49, gv91, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc49) gv92: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape99: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape98, gv92, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape98) gpt_neox_layers_0_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[5] gv93: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc50: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv93, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight5, gpt_neox_layers_0_post_attention_layernorm_bias5, alloc50) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias5) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[14] storage10: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv94: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc51: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv94, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5, alloc50, gpt_neox_layers_0_mlp_dense_h_to_4h_bias5, alloc51) R.vm.kill_object(alloc50) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias5) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[17] gv95: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc52: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv95, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5, alloc51, gpt_neox_layers_0_mlp_dense_4h_to_h_bias5, alloc52) R.vm.kill_object(alloc51) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias5) gpt_neox_layers_0_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[11] gv96: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc53: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv96, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight5, gpt_neox_layers_0_attention_dense_q_scale5, reshape99, gpt_neox_layers_0_attention_dense_bias5, alloc52, input_embeds, alloc53) R.vm.kill_object(reshape99) R.vm.kill_object(alloc52) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias5) gpt_neox_layers_1_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[19] gv97: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc54: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv97, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc53, gpt_neox_layers_1_input_layernorm_weight5, gpt_neox_layers_1_input_layernorm_bias5, alloc54) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias5) gpt_neox_layers_1_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[24] gv98: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc55: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv98, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_1_attention_query_key_value_q_weight5, gpt_neox_layers_1_attention_query_key_value_q_scale5, alloc54, gpt_neox_layers_1_attention_query_key_value_bias5, alloc55) R.vm.kill_object(alloc54) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias5) gv99: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape100: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc55, gv99, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc55) gv100: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape101: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape100, gv100, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape100) gv101: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc56: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv101, R.dtype("float16")) _55: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape101, alloc56) R.vm.kill_object(reshape101) gv102: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape102: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc56, gv102, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc56) gv103: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape103: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape102, gv103, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape102) gpt_neox_layers_1_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[21] gv104: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc57: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv104, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc53, gpt_neox_layers_1_post_attention_layernorm_weight5, gpt_neox_layers_1_post_attention_layernorm_bias5, alloc57) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias5) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[30] storage11: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv105: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc58: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv105, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5, alloc57, gpt_neox_layers_1_mlp_dense_h_to_4h_bias5, alloc58) R.vm.kill_object(alloc57) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias5) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[33] gv106: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc59: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv106, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5, alloc58, gpt_neox_layers_1_mlp_dense_4h_to_h_bias5, alloc59) R.vm.kill_object(alloc58) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias5) gpt_neox_layers_1_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[27] gv107: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc60: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv107, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_1_attention_dense_q_weight5, gpt_neox_layers_1_attention_dense_q_scale5, reshape103, gpt_neox_layers_1_attention_dense_bias5, alloc59, alloc53, alloc60) R.vm.kill_object(alloc53) R.vm.kill_object(reshape103) R.vm.kill_object(alloc59) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias5) gpt_neox_layers_2_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[35] gv108: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc61: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv108, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc60, gpt_neox_layers_2_input_layernorm_weight5, gpt_neox_layers_2_input_layernorm_bias5, alloc61) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias5) gpt_neox_layers_2_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[40] gv109: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc62: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv109, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_2_attention_query_key_value_q_weight5, gpt_neox_layers_2_attention_query_key_value_q_scale5, alloc61, gpt_neox_layers_2_attention_query_key_value_bias5, alloc62) R.vm.kill_object(alloc61) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias5) gv110: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape104: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc62, gv110, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc62) gv111: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape105: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape104, gv111, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape104) gv112: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc63: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv112, R.dtype("float16")) _62: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape105, alloc63) R.vm.kill_object(reshape105) gv113: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape106: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc63, gv113, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc63) gv114: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape107: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape106, gv114, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape106) gpt_neox_layers_2_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[37] gv115: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc64: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv115, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc60, gpt_neox_layers_2_post_attention_layernorm_weight5, gpt_neox_layers_2_post_attention_layernorm_bias5, alloc64) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias5) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[46] gv116: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc65: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv116, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5, alloc64, gpt_neox_layers_2_mlp_dense_h_to_4h_bias5, alloc65) R.vm.kill_object(alloc64) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias5) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[49] gv117: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc66: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv117, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5, alloc65, gpt_neox_layers_2_mlp_dense_4h_to_h_bias5, alloc66) R.vm.kill_object(alloc65) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias5) gpt_neox_layers_2_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[43] gv118: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc67: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv118, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_2_attention_dense_q_weight5, gpt_neox_layers_2_attention_dense_q_scale5, reshape107, gpt_neox_layers_2_attention_dense_bias5, alloc66, alloc60, alloc67) R.vm.kill_object(alloc60) R.vm.kill_object(reshape107) R.vm.kill_object(alloc66) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias5) gpt_neox_layers_3_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[51] gv119: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc68: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv119, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc67, gpt_neox_layers_3_input_layernorm_weight5, gpt_neox_layers_3_input_layernorm_bias5, alloc68) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias5) gpt_neox_layers_3_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[56] gv120: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc69: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv120, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_3_attention_query_key_value_q_weight5, gpt_neox_layers_3_attention_query_key_value_q_scale5, alloc68, gpt_neox_layers_3_attention_query_key_value_bias5, alloc69) R.vm.kill_object(alloc68) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias5) gv121: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape108: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc69, gv121, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc69) gv122: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape109: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape108, gv122, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape108) gv123: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc70: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv123, R.dtype("float16")) _69: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape109, alloc70) R.vm.kill_object(reshape109) gv124: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape110: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc70, gv124, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc70) gv125: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape111: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape110, gv125, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape110) gpt_neox_layers_3_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[53] gv126: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc71: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv126, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc67, gpt_neox_layers_3_post_attention_layernorm_weight5, gpt_neox_layers_3_post_attention_layernorm_bias5, alloc71) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias5) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[62] gv127: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc72: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv127, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5, alloc71, gpt_neox_layers_3_mlp_dense_h_to_4h_bias5, alloc72) R.vm.kill_object(alloc71) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias5) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[65] gv128: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc73: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv128, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5, alloc72, gpt_neox_layers_3_mlp_dense_4h_to_h_bias5, alloc73) R.vm.kill_object(alloc72) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias5) gpt_neox_layers_3_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[59] gv129: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc74: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv129, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_3_attention_dense_q_weight5, gpt_neox_layers_3_attention_dense_q_scale5, reshape111, gpt_neox_layers_3_attention_dense_bias5, alloc73, alloc67, alloc74) R.vm.kill_object(alloc67) R.vm.kill_object(reshape111) R.vm.kill_object(alloc73) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias5) gpt_neox_layers_4_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[67] gv130: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc75: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv130, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc74, gpt_neox_layers_4_input_layernorm_weight5, gpt_neox_layers_4_input_layernorm_bias5, alloc75) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias5) gpt_neox_layers_4_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[72] gv131: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc76: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv131, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_4_attention_query_key_value_q_weight5, gpt_neox_layers_4_attention_query_key_value_q_scale5, alloc75, gpt_neox_layers_4_attention_query_key_value_bias5, alloc76) R.vm.kill_object(alloc75) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias5) gv132: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape112: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc76, gv132, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc76) gv133: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape113: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape112, gv133, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape112) gv134: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc77: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv134, R.dtype("float16")) _76: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape113, alloc77) R.vm.kill_object(reshape113) gv135: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape114: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc77, gv135, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc77) gv136: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape115: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape114, gv136, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape114) gpt_neox_layers_4_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[69] gv137: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc78: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv137, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc74, gpt_neox_layers_4_post_attention_layernorm_weight5, gpt_neox_layers_4_post_attention_layernorm_bias5, alloc78) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias5) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[78] gv138: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc79: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv138, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5, alloc78, gpt_neox_layers_4_mlp_dense_h_to_4h_bias5, alloc79) R.vm.kill_object(alloc78) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias5) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[81] gv139: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc80: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv139, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5, alloc79, gpt_neox_layers_4_mlp_dense_4h_to_h_bias5, alloc80) R.vm.kill_object(alloc79) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias5) gpt_neox_layers_4_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[75] gv140: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc81: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv140, R.dtype("float16")) R.vm.kill_object(storage10) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_4_attention_dense_q_weight5, gpt_neox_layers_4_attention_dense_q_scale5, reshape115, gpt_neox_layers_4_attention_dense_bias5, alloc80, alloc74, alloc81) R.vm.kill_object(alloc74) R.vm.kill_object(reshape115) R.vm.kill_object(alloc80) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias5) gpt_neox_layers_5_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[83] gv141: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc82: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv141, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc81, gpt_neox_layers_5_input_layernorm_weight5, gpt_neox_layers_5_input_layernorm_bias5, alloc82) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias5) gpt_neox_layers_5_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[88] gv142: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc83: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv142, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_5_attention_query_key_value_q_weight5, gpt_neox_layers_5_attention_query_key_value_q_scale5, alloc82, gpt_neox_layers_5_attention_query_key_value_bias5, alloc83) R.vm.kill_object(alloc82) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias5) gv143: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape116: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc83, gv143, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc83) gv144: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape117: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape116, gv144, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape116) gv145: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc84: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv145, R.dtype("float16")) _83: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape117, alloc84) R.vm.kill_object(reshape117) gv146: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape118: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc84, gv146, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc84) gv147: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape119: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape118, gv147, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape118) gpt_neox_layers_5_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[85] gv148: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc85: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv148, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc81, gpt_neox_layers_5_post_attention_layernorm_weight5, gpt_neox_layers_5_post_attention_layernorm_bias5, alloc85) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias5) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[94] gv149: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc86: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv149, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5, alloc85, gpt_neox_layers_5_mlp_dense_h_to_4h_bias5, alloc86) R.vm.kill_object(alloc85) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias5) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[97] gv150: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc87: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv150, R.dtype("float16")) R.vm.kill_object(storage9) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5, alloc86, gpt_neox_layers_5_mlp_dense_4h_to_h_bias5, alloc87) R.vm.kill_object(alloc86) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias5) gpt_neox_layers_5_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[91] gv151: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc88: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv151, R.dtype("float16")) R.vm.kill_object(storage11) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_5_attention_dense_q_weight5, gpt_neox_layers_5_attention_dense_q_scale5, reshape119, gpt_neox_layers_5_attention_dense_bias5, alloc87, alloc81, alloc88) R.vm.kill_object(alloc81) R.vm.kill_object(reshape119) R.vm.kill_object(alloc87) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias5) gpt_neox_final_layer_norm_weight5: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias5: R.Tensor((512,), dtype="float16") = packed_params[99] gv152: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc89: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv152, R.dtype("float16")) R.vm.kill_object(storage8) cls.gpt_neox_q4f16_1_layer_norm1(alloc88, gpt_neox_final_layer_norm_weight5, gpt_neox_final_layer_norm_bias5, alloc89) R.vm.kill_object(alloc88) R.vm.kill_object(gpt_neox_final_layer_norm_weight5) R.vm.kill_object(gpt_neox_final_layer_norm_bias5) embed_out_q_weight5: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale5: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] gv153: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage12: R.Object = R.vm.alloc_storage(gv153, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv154: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc90: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage12, R.prim_value(0), gv154, R.dtype("float32")) R.vm.kill_object(storage12) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5(embed_out_q_weight5, embed_out_q_scale5, alloc89, alloc90) R.vm.kill_object(alloc89) R.vm.kill_object(embed_out_q_weight5) R.vm.kill_object(embed_out_q_scale5) gv5: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc90, paged_kv_cache R.vm.kill_object(alloc90) gv155: R.Tensor(dtype="float32", ndim=3) = gv5[0] R.call_packed("vm.builtin.match_shape", gv155, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(0), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=batch_verify, loc=return, annotation=R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv5 @R.function def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: max_batch_size = T.int64() max_total_seq_len = T.int64() prefill_chunk_size = T.int64() page_size = T.int64() support_sliding_window = T.int64() R.func_attr({"relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_shape_info", max_batch_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[0], param=max_batch_size_, annotation=R.Shape([max_batch_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", max_total_seq_len_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[1], param=max_total_seq_len_, annotation=R.Shape([max_total_seq_len])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", prefill_chunk_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[2], param=prefill_chunk_size_, annotation=R.Shape([prefill_chunk_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", page_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[3], param=page_size_, annotation=R.Shape([page_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", support_sliding_window_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[4], param=support_sliding_window_, annotation=R.Shape([support_sliding_window])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", max_batch_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(0), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[0], param=max_batch_size_, annotation=R.Shape([max_batch_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", max_total_seq_len_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[1], param=max_total_seq_len_, annotation=R.Shape([max_total_seq_len])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", prefill_chunk_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[2], param=prefill_chunk_size_, annotation=R.Shape([prefill_chunk_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", page_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[3], param=page_size_, annotation=R.Shape([page_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", support_sliding_window_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[4], param=support_sliding_window_, annotation=R.Shape([support_sliding_window])) "), sinfo_args=(R.Tuple,)) gv156: R.Shape(ndim=5) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(5), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(2), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=5),)) paged_kv_cache: R.Object = R.call_packed("vm.builtin.paged_attention_kv_cache_create_reduced", gv156, R.shape([0, 6]), R.prim_value(8), R.prim_value(8), R.prim_value(64), R.prim_value(1), R.prim_value(1), R.prim_value(10000), R.const(0.0, "float16"), cls.gpt_neox_q4f16_1_tir_kv_cache_transpose_append, cls.gpt_neox_q4f16_1_batch_prefill_paged_kv, cls.gpt_neox_q4f16_1_batch_decode_paged_kv, cls.gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window, cls.gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window, cls.gpt_neox_q4f16_1_batch_prefill_ragged_kv, cls.gpt_neox_q4f16_1_merge_state_inplace, cls.gpt_neox_q4f16_1_fused_rope, cls.gpt_neox_q4f16_1_copy_single_page, cls.gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv, cls.gpt_neox_q4f16_1_compact_kv_copy, cls.gpt_neox_q4f16_1_batch_tree_attn, cls.gpt_neox_q4f16_1_tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) return paged_kv_cache @R.function def decode(input_embed: R.Tensor((1, 1, 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(3),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embed, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[0], param=input_embed, annotation=R.Tensor((1, 1, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(102), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv157: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv157, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv158: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv158, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv159: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] R.call_packed("vm.builtin.check_tensor_info", gv159, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv160: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] R.call_packed("vm.builtin.check_tensor_info", gv160, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embed, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(512), R.str("ErrorContext(fn=decode, loc=param[0], param=input_embed, annotation=R.Tensor((1, 1, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv157, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv158, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv159, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv160, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func3(shape_heap) gpt_neox_layers_0_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[3] storage13: R.Object = R.vm.alloc_storage(R.shape([1024]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc91: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(input_embed, gpt_neox_layers_0_input_layernorm_weight2, gpt_neox_layers_0_input_layernorm_bias2, alloc91) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias2) gpt_neox_layers_0_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[8] storage14: R.Object = R.vm.alloc_storage(R.shape([3072]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc92: R.Tensor((1, 1, 1536), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1536]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_0_attention_query_key_value_q_weight2, gpt_neox_layers_0_attention_query_key_value_q_scale2, alloc91, gpt_neox_layers_0_attention_query_key_value_bias2, alloc92) R.vm.kill_object(alloc91) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias2) lv76: R.Tensor((1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc92, R.shape([1, 24, 64]), sinfo_args=(R.Tensor((1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc92) alloc93: R.Tensor((1, 8, 64), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 8, 64]), R.dtype("float16")) _92: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), lv76, alloc93) R.vm.kill_object(lv76) lv77: R.Tensor((1, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc93, R.shape([1, 1, 512]), sinfo_args=(R.Tensor((1, 1, 512), dtype="float16"),)) R.vm.kill_object(alloc93) gpt_neox_layers_0_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[5] alloc94: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight2, gpt_neox_layers_0_post_attention_layernorm_bias2, alloc94) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias2) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[14] storage15: R.Object = R.vm.alloc_storage(R.shape([4096]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc95: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, alloc94, gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, alloc95) R.vm.kill_object(alloc94) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias2) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[17] alloc96: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, alloc95, gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, alloc96) R.vm.kill_object(alloc95) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias2) gpt_neox_layers_0_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[11] alloc97: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_0_attention_dense_q_weight2, gpt_neox_layers_0_attention_dense_q_scale2, lv77, gpt_neox_layers_0_attention_dense_bias2, alloc96, input_embed, alloc97) R.vm.kill_object(lv77) R.vm.kill_object(alloc96) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias2) gpt_neox_layers_1_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[19] alloc98: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc97, gpt_neox_layers_1_input_layernorm_weight2, gpt_neox_layers_1_input_layernorm_bias2, alloc98) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias2) gpt_neox_layers_1_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[24] alloc99: R.Tensor((1, 1, 1536), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1536]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_1_attention_query_key_value_q_weight2, gpt_neox_layers_1_attention_query_key_value_q_scale2, alloc98, gpt_neox_layers_1_attention_query_key_value_bias2, alloc99) R.vm.kill_object(alloc98) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias2) lv82: R.Tensor((1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc99, R.shape([1, 24, 64]), sinfo_args=(R.Tensor((1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc99) alloc100: R.Tensor((1, 8, 64), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 8, 64]), R.dtype("float16")) _99: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), lv82, alloc100) R.vm.kill_object(lv82) lv83: R.Tensor((1, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc100, R.shape([1, 1, 512]), sinfo_args=(R.Tensor((1, 1, 512), dtype="float16"),)) R.vm.kill_object(alloc100) gpt_neox_layers_1_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[21] alloc101: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc97, gpt_neox_layers_1_post_attention_layernorm_weight2, gpt_neox_layers_1_post_attention_layernorm_bias2, alloc101) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias2) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[30] storage16: R.Object = R.vm.alloc_storage(R.shape([4096]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc102: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage16, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2, alloc101, gpt_neox_layers_1_mlp_dense_h_to_4h_bias2, alloc102) R.vm.kill_object(alloc101) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias2) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[33] alloc103: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2, alloc102, gpt_neox_layers_1_mlp_dense_4h_to_h_bias2, alloc103) R.vm.kill_object(alloc102) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias2) gpt_neox_layers_1_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[27] alloc104: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage16, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_1_attention_dense_q_weight2, gpt_neox_layers_1_attention_dense_q_scale2, lv83, gpt_neox_layers_1_attention_dense_bias2, alloc103, alloc97, alloc104) R.vm.kill_object(alloc97) R.vm.kill_object(lv83) R.vm.kill_object(alloc103) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias2) gpt_neox_layers_2_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[35] alloc105: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc104, gpt_neox_layers_2_input_layernorm_weight2, gpt_neox_layers_2_input_layernorm_bias2, alloc105) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias2) gpt_neox_layers_2_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[40] alloc106: R.Tensor((1, 1, 1536), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1536]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_2_attention_query_key_value_q_weight2, gpt_neox_layers_2_attention_query_key_value_q_scale2, alloc105, gpt_neox_layers_2_attention_query_key_value_bias2, alloc106) R.vm.kill_object(alloc105) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias2) lv88: R.Tensor((1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc106, R.shape([1, 24, 64]), sinfo_args=(R.Tensor((1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc106) alloc107: R.Tensor((1, 8, 64), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 8, 64]), R.dtype("float16")) _106: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), lv88, alloc107) R.vm.kill_object(lv88) lv89: R.Tensor((1, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc107, R.shape([1, 1, 512]), sinfo_args=(R.Tensor((1, 1, 512), dtype="float16"),)) R.vm.kill_object(alloc107) gpt_neox_layers_2_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[37] alloc108: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc104, gpt_neox_layers_2_post_attention_layernorm_weight2, gpt_neox_layers_2_post_attention_layernorm_bias2, alloc108) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias2) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[46] alloc109: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2, alloc108, gpt_neox_layers_2_mlp_dense_h_to_4h_bias2, alloc109) R.vm.kill_object(alloc108) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias2) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[49] alloc110: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2, alloc109, gpt_neox_layers_2_mlp_dense_4h_to_h_bias2, alloc110) R.vm.kill_object(alloc109) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias2) gpt_neox_layers_2_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[43] alloc111: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_2_attention_dense_q_weight2, gpt_neox_layers_2_attention_dense_q_scale2, lv89, gpt_neox_layers_2_attention_dense_bias2, alloc110, alloc104, alloc111) R.vm.kill_object(alloc104) R.vm.kill_object(lv89) R.vm.kill_object(alloc110) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias2) gpt_neox_layers_3_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[51] alloc112: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc111, gpt_neox_layers_3_input_layernorm_weight2, gpt_neox_layers_3_input_layernorm_bias2, alloc112) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias2) gpt_neox_layers_3_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[56] alloc113: R.Tensor((1, 1, 1536), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1536]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_3_attention_query_key_value_q_weight2, gpt_neox_layers_3_attention_query_key_value_q_scale2, alloc112, gpt_neox_layers_3_attention_query_key_value_bias2, alloc113) R.vm.kill_object(alloc112) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias2) lv94: R.Tensor((1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc113, R.shape([1, 24, 64]), sinfo_args=(R.Tensor((1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc113) alloc114: R.Tensor((1, 8, 64), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 8, 64]), R.dtype("float16")) _113: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), lv94, alloc114) R.vm.kill_object(lv94) lv95: R.Tensor((1, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc114, R.shape([1, 1, 512]), sinfo_args=(R.Tensor((1, 1, 512), dtype="float16"),)) R.vm.kill_object(alloc114) gpt_neox_layers_3_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[53] alloc115: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc111, gpt_neox_layers_3_post_attention_layernorm_weight2, gpt_neox_layers_3_post_attention_layernorm_bias2, alloc115) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias2) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[62] alloc116: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage16, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2, alloc115, gpt_neox_layers_3_mlp_dense_h_to_4h_bias2, alloc116) R.vm.kill_object(alloc115) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias2) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[65] alloc117: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2, alloc116, gpt_neox_layers_3_mlp_dense_4h_to_h_bias2, alloc117) R.vm.kill_object(alloc116) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias2) gpt_neox_layers_3_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[59] alloc118: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage16, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_3_attention_dense_q_weight2, gpt_neox_layers_3_attention_dense_q_scale2, lv95, gpt_neox_layers_3_attention_dense_bias2, alloc117, alloc111, alloc118) R.vm.kill_object(alloc111) R.vm.kill_object(lv95) R.vm.kill_object(alloc117) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias2) gpt_neox_layers_4_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[67] alloc119: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc118, gpt_neox_layers_4_input_layernorm_weight2, gpt_neox_layers_4_input_layernorm_bias2, alloc119) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias2) gpt_neox_layers_4_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[72] alloc120: R.Tensor((1, 1, 1536), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1536]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_4_attention_query_key_value_q_weight2, gpt_neox_layers_4_attention_query_key_value_q_scale2, alloc119, gpt_neox_layers_4_attention_query_key_value_bias2, alloc120) R.vm.kill_object(alloc119) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias2) lv100: R.Tensor((1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc120, R.shape([1, 24, 64]), sinfo_args=(R.Tensor((1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc120) alloc121: R.Tensor((1, 8, 64), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 8, 64]), R.dtype("float16")) _120: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), lv100, alloc121) R.vm.kill_object(lv100) lv101: R.Tensor((1, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc121, R.shape([1, 1, 512]), sinfo_args=(R.Tensor((1, 1, 512), dtype="float16"),)) R.vm.kill_object(alloc121) gpt_neox_layers_4_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[69] alloc122: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc118, gpt_neox_layers_4_post_attention_layernorm_weight2, gpt_neox_layers_4_post_attention_layernorm_bias2, alloc122) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias2) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[78] alloc123: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2, alloc122, gpt_neox_layers_4_mlp_dense_h_to_4h_bias2, alloc123) R.vm.kill_object(alloc122) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias2) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[81] alloc124: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2, alloc123, gpt_neox_layers_4_mlp_dense_4h_to_h_bias2, alloc124) R.vm.kill_object(alloc123) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias2) gpt_neox_layers_4_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[75] alloc125: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) R.vm.kill_object(storage15) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_4_attention_dense_q_weight2, gpt_neox_layers_4_attention_dense_q_scale2, lv101, gpt_neox_layers_4_attention_dense_bias2, alloc124, alloc118, alloc125) R.vm.kill_object(alloc118) R.vm.kill_object(lv101) R.vm.kill_object(alloc124) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias2) gpt_neox_layers_5_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[83] alloc126: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc125, gpt_neox_layers_5_input_layernorm_weight2, gpt_neox_layers_5_input_layernorm_bias2, alloc126) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias2) gpt_neox_layers_5_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[88] alloc127: R.Tensor((1, 1, 1536), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1536]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_5_attention_query_key_value_q_weight2, gpt_neox_layers_5_attention_query_key_value_q_scale2, alloc126, gpt_neox_layers_5_attention_query_key_value_bias2, alloc127) R.vm.kill_object(alloc126) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias2) lv106: R.Tensor((1, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc127, R.shape([1, 24, 64]), sinfo_args=(R.Tensor((1, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc127) alloc128: R.Tensor((1, 8, 64), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 8, 64]), R.dtype("float16")) _127: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), lv106, alloc128) R.vm.kill_object(lv106) lv107: R.Tensor((1, 1, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc128, R.shape([1, 1, 512]), sinfo_args=(R.Tensor((1, 1, 512), dtype="float16"),)) R.vm.kill_object(alloc128) gpt_neox_layers_5_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[85] alloc129: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc125, gpt_neox_layers_5_post_attention_layernorm_weight2, gpt_neox_layers_5_post_attention_layernorm_bias2, alloc129) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias2) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[94] alloc130: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage16, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2, alloc129, gpt_neox_layers_5_mlp_dense_h_to_4h_bias2, alloc130) R.vm.kill_object(alloc129) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias2) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[97] alloc131: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) R.vm.kill_object(storage14) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2, alloc130, gpt_neox_layers_5_mlp_dense_4h_to_h_bias2, alloc131) R.vm.kill_object(alloc130) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias2) gpt_neox_layers_5_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[91] alloc132: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage16, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) R.vm.kill_object(storage16) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_5_attention_dense_q_weight2, gpt_neox_layers_5_attention_dense_q_scale2, lv107, gpt_neox_layers_5_attention_dense_bias2, alloc131, alloc125, alloc132) R.vm.kill_object(alloc125) R.vm.kill_object(lv107) R.vm.kill_object(alloc131) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias2) gpt_neox_final_layer_norm_weight2: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias2: R.Tensor((512,), dtype="float16") = packed_params[99] alloc133: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) R.vm.kill_object(storage13) cls.gpt_neox_q4f16_1_layer_norm2(alloc132, gpt_neox_final_layer_norm_weight2, gpt_neox_final_layer_norm_bias2, alloc133) R.vm.kill_object(alloc132) R.vm.kill_object(gpt_neox_final_layer_norm_weight2) R.vm.kill_object(gpt_neox_final_layer_norm_bias2) embed_out_q_weight2: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale2: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] gv161: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage17: R.Object = R.vm.alloc_storage(gv161, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv162: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), sinfo_args=(R.Shape(ndim=3),)) alloc134: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv162, R.dtype("float32")) R.vm.kill_object(storage17) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8(embed_out_q_weight2, embed_out_q_scale2, alloc133, alloc134) R.vm.kill_object(alloc133) R.vm.kill_object(embed_out_q_weight2) R.vm.kill_object(embed_out_q_scale2) gv2: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc134, paged_kv_cache R.vm.kill_object(alloc134) gv163: R.Tensor(dtype="float32", ndim=3) = gv2[0] R.call_packed("vm.builtin.match_shape", gv163, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(0), R.str("ErrorContext(fn=decode, loc=return, annotation=R.Tuple(R.Tensor((1, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv2 @R.function def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tensor(("seq_len", 512), dtype="float16"): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(2),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_ids, R.prim_value(1), R.dtype("int32"), R.str("ErrorContext(fn=embed, loc=param[0], param=input_ids, annotation=R.Tensor((seq_len,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(102), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv238: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv238, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv239: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv239, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv240: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] R.call_packed("vm.builtin.check_tensor_info", gv240, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv241: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] R.call_packed("vm.builtin.check_tensor_info", gv241, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_ids, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(0), R.str("ErrorContext(fn=embed, loc=param[0], param=input_ids, annotation=R.Tensor((seq_len,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv238, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv239, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv240, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv241, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gpt_neox_embed_in_q_weight: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] gpt_neox_embed_in_q_scale: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] storage24: R.Object = R.vm.alloc_storage(R.shape([2097152]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv242: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=2),)) alloc180: R.Tensor(dtype="float16", ndim=2) = R.vm.alloc_tensor(storage24, R.prim_value(0), gv242, R.dtype("float16")) R.vm.kill_object(storage24) cls.gpt_neox_q4f16_1_fused_dequantize_take1(gpt_neox_embed_in_q_weight, gpt_neox_embed_in_q_scale, input_ids, alloc180) R.vm.kill_object(gpt_neox_embed_in_q_weight) R.vm.kill_object(gpt_neox_embed_in_q_scale) R.call_packed("vm.builtin.match_shape", alloc180, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(512), R.str("ErrorContext(fn=embed, loc=return, annotation=R.Tensor((seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) return alloc180 @R.function def prefill(input_embed: R.Tensor((1, "seq_len", 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embed, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[0], param=input_embed, annotation=R.Tensor((1, seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(102), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv164: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv164, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv165: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv165, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv166: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] R.call_packed("vm.builtin.check_tensor_info", gv166, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv167: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] R.call_packed("vm.builtin.check_tensor_info", gv167, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embed, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), R.str("ErrorContext(fn=prefill, loc=param[0], param=input_embed, annotation=R.Tensor((1, seq_len, 512), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv164, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv165, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv166, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(64), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv167, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((1536, 64), dtype=\"uint32\"), R.Tensor((1536, 16), dtype=\"float16\"), R.Tensor((1536,), dtype=\"float16\"), R.Tensor((512, 64), dtype=\"uint32\"), R.Tensor((512, 16), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((2048, 64), dtype=\"uint32\"), R.Tensor((2048, 16), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float32\"), R.Tensor((512, 256), dtype=\"uint32\"), R.Tensor((512, 64), dtype=\"float16\"), R.Tensor((512,), dtype=\"float32\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((512,), dtype=\"float16\"), R.Tensor((vocab_size, 64), dtype=\"uint32\"), R.Tensor((vocab_size, 16), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func4(shape_heap) gpt_neox_layers_0_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[3] storage18: R.Object = R.vm.alloc_storage(R.shape([2097152]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv168: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc135: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv168, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embed, gpt_neox_layers_0_input_layernorm_weight1, gpt_neox_layers_0_input_layernorm_bias1, alloc135) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias1) gpt_neox_layers_0_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[8] storage19: R.Object = R.vm.alloc_storage(R.shape([6291456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv169: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc136: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv169, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight1, gpt_neox_layers_0_attention_query_key_value_q_scale1, alloc135, gpt_neox_layers_0_attention_query_key_value_bias1, alloc136) R.vm.kill_object(alloc135) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias1) gv170: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc136, gv170, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc136) gv171: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape1: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape, gv171, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape) gv172: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc137: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv172, R.dtype("float16")) _136: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1, alloc137) R.vm.kill_object(reshape1) gv173: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape2: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc137, gv173, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc137) gv174: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape3: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape2, gv174, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape2) gpt_neox_layers_0_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[5] gv175: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc138: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv175, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight1, gpt_neox_layers_0_post_attention_layernorm_bias1, alloc138) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias1) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[14] storage20: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv176: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc139: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv176, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1, alloc138, gpt_neox_layers_0_mlp_dense_h_to_4h_bias1, alloc139) R.vm.kill_object(alloc138) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias1) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[17] gv177: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc140: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv177, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1, alloc139, gpt_neox_layers_0_mlp_dense_4h_to_h_bias1, alloc140) R.vm.kill_object(alloc139) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias1) gpt_neox_layers_0_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[11] gv178: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc141: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv178, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight1, gpt_neox_layers_0_attention_dense_q_scale1, reshape3, gpt_neox_layers_0_attention_dense_bias1, alloc140, input_embed, alloc141) R.vm.kill_object(reshape3) R.vm.kill_object(alloc140) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias1) gpt_neox_layers_1_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[19] gv179: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc142: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv179, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc141, gpt_neox_layers_1_input_layernorm_weight1, gpt_neox_layers_1_input_layernorm_bias1, alloc142) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias1) gpt_neox_layers_1_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[24] gv180: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc143: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv180, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_1_attention_query_key_value_q_weight1, gpt_neox_layers_1_attention_query_key_value_q_scale1, alloc142, gpt_neox_layers_1_attention_query_key_value_bias1, alloc143) R.vm.kill_object(alloc142) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias1) gv181: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape4: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc143, gv181, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc143) gv182: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape5: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape4, gv182, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape4) gv183: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc144: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv183, R.dtype("float16")) _143: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5, alloc144) R.vm.kill_object(reshape5) gv184: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape6: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc144, gv184, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc144) gv185: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape7: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape6, gv185, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape6) gpt_neox_layers_1_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[21] gv186: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc145: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv186, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc141, gpt_neox_layers_1_post_attention_layernorm_weight1, gpt_neox_layers_1_post_attention_layernorm_bias1, alloc145) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias1) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[30] storage21: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv187: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc146: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv187, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1, alloc145, gpt_neox_layers_1_mlp_dense_h_to_4h_bias1, alloc146) R.vm.kill_object(alloc145) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias1) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[33] gv188: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc147: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv188, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1, alloc146, gpt_neox_layers_1_mlp_dense_4h_to_h_bias1, alloc147) R.vm.kill_object(alloc146) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias1) gpt_neox_layers_1_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[27] gv189: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc148: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv189, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_1_attention_dense_q_weight1, gpt_neox_layers_1_attention_dense_q_scale1, reshape7, gpt_neox_layers_1_attention_dense_bias1, alloc147, alloc141, alloc148) R.vm.kill_object(alloc141) R.vm.kill_object(reshape7) R.vm.kill_object(alloc147) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias1) gpt_neox_layers_2_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[35] gv190: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc149: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv190, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc148, gpt_neox_layers_2_input_layernorm_weight1, gpt_neox_layers_2_input_layernorm_bias1, alloc149) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias1) gpt_neox_layers_2_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[40] gv191: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc150: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv191, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_2_attention_query_key_value_q_weight1, gpt_neox_layers_2_attention_query_key_value_q_scale1, alloc149, gpt_neox_layers_2_attention_query_key_value_bias1, alloc150) R.vm.kill_object(alloc149) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias1) gv192: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape8: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc150, gv192, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc150) gv193: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape9: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape8, gv193, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape8) gv194: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc151: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv194, R.dtype("float16")) _150: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9, alloc151) R.vm.kill_object(reshape9) gv195: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape10: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc151, gv195, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc151) gv196: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape11: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape10, gv196, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape10) gpt_neox_layers_2_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[37] gv197: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc152: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv197, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc148, gpt_neox_layers_2_post_attention_layernorm_weight1, gpt_neox_layers_2_post_attention_layernorm_bias1, alloc152) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias1) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[46] gv198: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc153: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv198, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1, alloc152, gpt_neox_layers_2_mlp_dense_h_to_4h_bias1, alloc153) R.vm.kill_object(alloc152) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias1) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[49] gv199: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc154: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv199, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1, alloc153, gpt_neox_layers_2_mlp_dense_4h_to_h_bias1, alloc154) R.vm.kill_object(alloc153) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias1) gpt_neox_layers_2_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[43] gv200: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc155: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv200, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_2_attention_dense_q_weight1, gpt_neox_layers_2_attention_dense_q_scale1, reshape11, gpt_neox_layers_2_attention_dense_bias1, alloc154, alloc148, alloc155) R.vm.kill_object(alloc148) R.vm.kill_object(reshape11) R.vm.kill_object(alloc154) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias1) gpt_neox_layers_3_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[51] gv201: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc156: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv201, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc155, gpt_neox_layers_3_input_layernorm_weight1, gpt_neox_layers_3_input_layernorm_bias1, alloc156) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias1) gpt_neox_layers_3_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[56] gv202: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc157: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv202, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_3_attention_query_key_value_q_weight1, gpt_neox_layers_3_attention_query_key_value_q_scale1, alloc156, gpt_neox_layers_3_attention_query_key_value_bias1, alloc157) R.vm.kill_object(alloc156) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias1) gv203: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape12: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc157, gv203, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc157) gv204: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape13: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape12, gv204, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape12) gv205: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc158: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv205, R.dtype("float16")) _157: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13, alloc158) R.vm.kill_object(reshape13) gv206: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape14: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc158, gv206, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc158) gv207: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape15: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape14, gv207, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape14) gpt_neox_layers_3_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[53] gv208: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc159: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv208, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc155, gpt_neox_layers_3_post_attention_layernorm_weight1, gpt_neox_layers_3_post_attention_layernorm_bias1, alloc159) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias1) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[62] gv209: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc160: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv209, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1, alloc159, gpt_neox_layers_3_mlp_dense_h_to_4h_bias1, alloc160) R.vm.kill_object(alloc159) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias1) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[65] gv210: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc161: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv210, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1, alloc160, gpt_neox_layers_3_mlp_dense_4h_to_h_bias1, alloc161) R.vm.kill_object(alloc160) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias1) gpt_neox_layers_3_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[59] gv211: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc162: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv211, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_3_attention_dense_q_weight1, gpt_neox_layers_3_attention_dense_q_scale1, reshape15, gpt_neox_layers_3_attention_dense_bias1, alloc161, alloc155, alloc162) R.vm.kill_object(alloc155) R.vm.kill_object(reshape15) R.vm.kill_object(alloc161) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias1) gpt_neox_layers_4_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[67] gv212: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc163: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv212, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc162, gpt_neox_layers_4_input_layernorm_weight1, gpt_neox_layers_4_input_layernorm_bias1, alloc163) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias1) gpt_neox_layers_4_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[72] gv213: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc164: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv213, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_4_attention_query_key_value_q_weight1, gpt_neox_layers_4_attention_query_key_value_q_scale1, alloc163, gpt_neox_layers_4_attention_query_key_value_bias1, alloc164) R.vm.kill_object(alloc163) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias1) gv214: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape16: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc164, gv214, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc164) gv215: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape17: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape16, gv215, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape16) gv216: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc165: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv216, R.dtype("float16")) _164: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17, alloc165) R.vm.kill_object(reshape17) gv217: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape18: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc165, gv217, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc165) gv218: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape19: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape18, gv218, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape18) gpt_neox_layers_4_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[69] gv219: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc166: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv219, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc162, gpt_neox_layers_4_post_attention_layernorm_weight1, gpt_neox_layers_4_post_attention_layernorm_bias1, alloc166) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias1) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[78] gv220: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc167: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv220, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1, alloc166, gpt_neox_layers_4_mlp_dense_h_to_4h_bias1, alloc167) R.vm.kill_object(alloc166) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias1) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[81] gv221: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc168: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv221, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1, alloc167, gpt_neox_layers_4_mlp_dense_4h_to_h_bias1, alloc168) R.vm.kill_object(alloc167) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias1) gpt_neox_layers_4_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[75] gv222: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc169: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv222, R.dtype("float16")) R.vm.kill_object(storage20) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_4_attention_dense_q_weight1, gpt_neox_layers_4_attention_dense_q_scale1, reshape19, gpt_neox_layers_4_attention_dense_bias1, alloc168, alloc162, alloc169) R.vm.kill_object(alloc162) R.vm.kill_object(reshape19) R.vm.kill_object(alloc168) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias1) gpt_neox_layers_5_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[83] gv223: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc170: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv223, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc169, gpt_neox_layers_5_input_layernorm_weight1, gpt_neox_layers_5_input_layernorm_bias1, alloc170) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias1) gpt_neox_layers_5_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[88] gv224: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1536), sinfo_args=(R.Shape(ndim=3),)) alloc171: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv224, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_5_attention_query_key_value_q_weight1, gpt_neox_layers_5_attention_query_key_value_q_scale1, alloc170, gpt_neox_layers_5_attention_query_key_value_bias1, alloc171) R.vm.kill_object(alloc170) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias1) gv225: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape20: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc171, gv225, sinfo_args=(R.Tensor((1, seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(alloc171) gv226: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(24), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape21: R.Tensor((seq_len, 24, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape20, gv226, sinfo_args=(R.Tensor((seq_len, 24, 64), dtype="float16"),)) R.vm.kill_object(reshape20) gv227: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc172: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv227, R.dtype("float16")) _171: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21, alloc172) R.vm.kill_object(reshape21) gv228: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape22: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc172, gv228, sinfo_args=(R.Tensor((1, seq_len, 8, 64), dtype="float16"),)) R.vm.kill_object(alloc172) gv229: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) reshape23: R.Tensor((1, seq_len, 512), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape22, gv229, sinfo_args=(R.Tensor((1, seq_len, 512), dtype="float16"),)) R.vm.kill_object(reshape22) gpt_neox_layers_5_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[85] gv230: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc173: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv230, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc169, gpt_neox_layers_5_post_attention_layernorm_weight1, gpt_neox_layers_5_post_attention_layernorm_bias1, alloc173) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias1) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[94] gv231: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc174: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv231, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1, alloc173, gpt_neox_layers_5_mlp_dense_h_to_4h_bias1, alloc174) R.vm.kill_object(alloc173) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias1) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[97] gv232: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc175: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv232, R.dtype("float16")) R.vm.kill_object(storage19) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1, alloc174, gpt_neox_layers_5_mlp_dense_4h_to_h_bias1, alloc175) R.vm.kill_object(alloc174) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias1) gpt_neox_layers_5_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[91] gv233: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc176: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv233, R.dtype("float16")) R.vm.kill_object(storage21) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_5_attention_dense_q_weight1, gpt_neox_layers_5_attention_dense_q_scale1, reshape23, gpt_neox_layers_5_attention_dense_bias1, alloc175, alloc169, alloc176) R.vm.kill_object(alloc169) R.vm.kill_object(reshape23) R.vm.kill_object(alloc175) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias1) gpt_neox_final_layer_norm_weight1: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias1: R.Tensor((512,), dtype="float16") = packed_params[99] gv234: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(512), sinfo_args=(R.Shape(ndim=3),)) alloc177: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv234, R.dtype("float16")) R.vm.kill_object(storage18) cls.gpt_neox_q4f16_1_layer_norm1(alloc176, gpt_neox_final_layer_norm_weight1, gpt_neox_final_layer_norm_bias1, alloc177) R.vm.kill_object(alloc176) R.vm.kill_object(gpt_neox_final_layer_norm_weight1) R.vm.kill_object(gpt_neox_final_layer_norm_bias1) storage22: R.Object = R.vm.alloc_storage(R.shape([1024]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc178: R.Tensor((1, 1, 512), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 512]), R.dtype("float16")) R.vm.kill_object(storage22) cls.gpt_neox_q4f16_1_index(alloc177, alloc178) R.vm.kill_object(alloc177) embed_out_q_weight1: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale1: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] gv235: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage23: R.Object = R.vm.alloc_storage(gv235, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv236: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc179: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv236, R.dtype("float32")) R.vm.kill_object(storage23) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8(embed_out_q_weight1, embed_out_q_scale1, alloc178, alloc179) R.vm.kill_object(alloc178) R.vm.kill_object(embed_out_q_weight1) R.vm.kill_object(embed_out_q_scale1) gv1: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc179, paged_kv_cache R.vm.kill_object(alloc179) gv237: R.Tensor(dtype="float32", ndim=3) = gv1[0] R.call_packed("vm.builtin.match_shape", gv237, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=prefill, loc=return, annotation=R.Tuple(R.Tensor((1, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv1 @R.function def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64(is_size_var=True) vocab_size = T.int64(is_size_var=True) R.func_attr({"relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", logits, R.prim_value(3), R.dtype("float32"), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[0], param=logits, annotation=R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tensor_info", temperature, R.prim_value(1), R.dtype("float32"), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[1], param=temperature, annotation=R.Tensor((batch_size,), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", logits, shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[0], param=logits, annotation=R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", temperature, shape_heap, R.prim_value(1), R.prim_value(3), R.prim_value(0), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[1], param=temperature, annotation=R.Tensor((batch_size,), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func(shape_heap) gv: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=2),)) lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_packed("vm.builtin.reshape", logits, gv, sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) gv1: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage: R.Object = R.vm.alloc_storage(gv1, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv2: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=2),)) alloc: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage, R.prim_value(0), gv2, R.dtype("float32")) R.vm.kill_object(storage) gv3: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage1: R.Object = R.vm.alloc_storage(gv3, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv4: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=2),)) alloc1: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv4, R.dtype("float32")) R.vm.kill_object(storage1) cls.gpt_neox_q4f16_1_chunk_lse(lv, temperature, alloc, alloc1) lv1: R.Tuple(R.Tensor(dtype="float32", ndim=2), R.Tensor(dtype="float32", ndim=2)) = alloc, alloc1 gv5: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=1),)) storage2: R.Object = R.vm.alloc_storage(gv5, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv6: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=2),)) alloc2: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv6, R.dtype("float32")) R.vm.kill_object(storage2) cls.gpt_neox_q4f16_1_softmax_with_chunked_sum(lv, temperature, alloc, alloc1, alloc2) R.vm.kill_object(lv) R.vm.kill_object(alloc) R.vm.kill_object(alloc1) gv7: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) gv_1: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_packed("vm.builtin.reshape", alloc2, gv7, sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) R.vm.kill_object(alloc2) return gv_1