# from tvm.script import ir as I # from tvm.script import tir as T # from tvm.script import relax as R @I.ir_module class Module: I.module_attrs({"system_lib_prefix": "gpt_neox_q4f16_1_"}) @T.prim_func def gpt_neox_q4f16_1_apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") # with T.block("root"): for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) T.writes(logits[seq_ids[vs], vv]) logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) @T.prim_func def gpt_neox_q4f16_1_apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") logit_bias = T.match_buffer(var_logit_bias, (num_token,)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) T.writes(logits[pos2seq_id[vp], token_ids[vp]]) logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] @T.prim_func def gpt_neox_q4f16_1_apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") penalties = T.match_buffer(var_penalties, (num_seq, 3)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) @T.prim_func def gpt_neox_q4f16_1_batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 16, 64), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 16, 16, 64), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 16, 64), "float16") lse = T.match_buffer(lse_handle, (B, 16)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.18033688011112042) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(16, thread="blockIdx.y"): for ty in T.thread_binding(1, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): for tz in T.thread_binding(16, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 16 + ty + fused_by_bz % 16, tx * 4 - 32:tx * 4 - 32 + 68]) T.writes(output[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared") md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((1,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = fused_by_bz % 16 bz: T.int32 = fused_by_bz // 16 batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 15) // 16): tile_start_s: T.int32 = tz + ty tile_start_g: T.int32 = iterator * 16 + tz + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = row_g page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(1): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if iterator * 16 + tz + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(1): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(1): for vec in T.vectorized(4): V_local[vec] = V_smem[tz + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] md_allreduce[tz, ty, 0] = st_m[0] md_allreduce[tz, ty, 1] = st_d[0] T.tvm_storage_sync("shared") st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for j in range(16): m_prev[0] = st_m[0] d_prev[0] = st_d[0] other_m[0] = md_allreduce[j, ty, 0] other_d[0] = md_allreduce[j, ty, 1] for vec in T.vectorized(4): other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] st_m[0] = T.max(st_m[0], other_m[0]) st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) for vec in T.vectorized(4): O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 16, 64), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 16, 16, 64), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 16, 64), "float16") lse = T.match_buffer(lse_handle, (B, 16)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.18033688011112042) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(16, thread="blockIdx.y"): for ty in T.thread_binding(1, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): for tz in T.thread_binding(16, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz // 16 + ty + fused_by_bz % 16, tx * 4 - 32:tx * 4 - 32 + 68]) T.writes(output[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared") md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((1,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = fused_by_bz % 16 bz: T.int32 = fused_by_bz // 16 batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 15) // 16): tile_start_s: T.int32 = tz + ty tile_start_g: T.int32 = iterator * 16 + tz + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(1): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if iterator * 16 + tz + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(1): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(1): for vec in T.vectorized(4): V_local[vec] = V_smem[tz + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] md_allreduce[tz, ty, 0] = st_m[0] md_allreduce[tz, ty, 1] = st_d[0] T.tvm_storage_sync("shared") st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for j in range(16): m_prev[0] = st_m[0] d_prev[0] = st_d[0] other_m[0] = md_allreduce[j, ty, 0] other_d[0] = md_allreduce[j, ty, 1] for vec in T.vectorized(4): other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] st_m[0] = T.max(st_m[0], other_m[0]) st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) for vec in T.vectorized(4): O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def gpt_neox_q4f16_1_batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 16, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 16, 16, 64), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 16, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 16)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(16, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 16), scope="shared") S_local = T.alloc_buffer((32, 16), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64]) T.writes(S_local[0:32, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 16, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 16, 16, 64), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 16, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 16)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(16, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 16), scope="shared") S_local = T.alloc_buffer((32, 16), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64]) T.writes(S_local[0:32, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gpt_neox_q4f16_1_batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 16, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 16, 64), "float16") v = T.match_buffer(var_v, (kv_len, 16, 64), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 16, 64), "float16") lse = T.match_buffer(var_lse, (qo_len, 16)) # with T.block("root"): for lbx in T.thread_binding(8, thread="blockIdx.x"): for lby in T.thread_binding(16, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((64, 32), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] q_indptr_val: T.int32 = q_indptr[b_idx] LH_start: T.int32 = tile_id[0] * 32 kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_0, lj_0_0 in T.grid(1, 2): for lj_1 in T.vectorized(8): with T.block("Q_load"): i = T.axis.spatial(32, li_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 4) j = T.axis.spatial(64, lj_0_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 4 * 8 + lj_1) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(1, 2): for ly_1 in T.vectorized(8): with T.block("K_load"): i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4) j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1) T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 32:j - 32 + 65]) T.writes(K_smem[j, i]) cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: freq = T.float32() K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[L_kv_base + cur_L, by, j + 32] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[L_kv_base + cur_L, by, j]) else: K_smem[j, i] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(1, 2): for ly_1 in T.vectorized(8): with T.block("V_load"): i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4) j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: V_smem[i, j] = v[L_kv_base + cur_L, by, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:64, 0:32]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(2): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_0_init * 4 + lj_1_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0 in range(4): for li_1 in T.unroll(2): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): for lk_1 in range(16): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) k_1 = T.axis.reduce(64, lk_0 * 16 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(2): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(4): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_0_init * 4 + lj_1_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1 in T.grid(2, 16): for li_1 in T.unroll(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) k_1 = T.axis.reduce(32, lk_0 * 16 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 8 @T.prim_func def gpt_neox_q4f16_1_batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 16, 64), "float16") q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 16, 64), "float16") v = T.match_buffer(var_v, (kv_len, 16, 64), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) tree_size = T.int32(is_size_var=True) mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 16, 64), "float16") lse = T.match_buffer(var_lse, (qo_len, 16)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(16, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((32, 64), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("KV_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_base + L_kv_start + i if L_kv_start + i < kv_chunk_len[0]: freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[cur_L, by, j + 32] * T.float16(-1.0), k[cur_L, by, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[cur_L, by, j]) V_smem[i, j] = v[cur_L, by, j] else: K_smem[i, j] = T.float16(0.0) V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) k_1 = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gpt_neox_q4f16_1_chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int32(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size, num_chunks), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size, num_chunks), scope="shared") for ax0_ax1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0, ax1 in T.grid(1, 1): for ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks + ax0) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks + ax1) v2 = T.axis.reduce(4096, ax2_fused_0 * 256 + ax2_fused_1) T.reads(temperature[v0], A[v0, v1 * 4096 + v2]) T.writes(temp_max_shared[v0, v1]) with T.init(): temp_max_shared[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0, v1] = T.max(temp_max_shared[v0, v1], T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0))) for ax0, ax1 in T.grid(1, 1): for ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks + ax0) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks + ax1) v2 = T.axis.reduce(4096, ax2_fused_0 * 256 + ax2_fused_1) T.reads(temperature[v0], A[v0, v1 * 4096 + v2], temp_max_shared[v0, v1]) T.writes(temp_sum_shared[v0, v1]) with T.init(): temp_sum_shared[v0, v1] = T.float32(0.0) temp_sum_shared[v0, v1] = temp_sum_shared[v0, v1] + T.if_then_else(v1 * 4096 + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0)) - temp_max_shared[v0, v1]), T.Cast("float32", T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0)) == temp_max_shared[v0, v1])), T.float32(0.0)) for ax2_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("log"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks) v2 = T.axis.spatial(1, ax2_0 * 256 + ax2_1) T.where(ax2_0 * 256 + ax2_1 < 1) T.reads(temperature[v0], temp_sum_shared[v0, v1], temp_max_shared[v0, v1]) T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum_shared[v0, v1]), temp_sum_shared[v0, v1]) chunked_max[v0, v1] = temp_max_shared[v0, v1] @T.prim_func def gpt_neox_q4f16_1_compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 16, 16, 64), "float16", offset_factor=1) copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) total_copy_length = T.int32() copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) with T.block("root"): T.reads() T.writes() for bhd_o in T.thread_binding(batch_size * 4, thread="blockIdx.x"): for bhd_i in T.thread_binding(256, thread="threadIdx.x"): b: T.int32 = (bhd_o * 256 + bhd_i) // 1024 h: T.int32 = (bhd_o * 256 + bhd_i) // 64 % 16 d: T.int32 = (bhd_o * 256 + bhd_i) % 64 if bhd_o * 256 + bhd_i < batch_size * 16 * 64: for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] @T.prim_func def gpt_neox_q4f16_1_copy_single_page(var_pages: T.handle, src_page_id: T.int32, tgt_page_id: T.int32, copy_length: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages, page_size = T.int32(), T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 16, page_size, 64), "float16", offset_factor=1) # with T.block("root"): for b in T.thread_binding(copy_length * 4, thread="blockIdx.x"): for t in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy"): vh = T.axis.spatial(16, (b * 256 + t) // (copy_length * 64)) vp = T.axis.spatial(copy_length, (b * 256 + t) % (copy_length * 64) // 64) vd = T.axis.spatial(64, (b * 256 + t) % 64) T.where(b * 256 + t < copy_length * 16 * 64) T.reads(pages[src_page_id, 0:2, vh, vp, vd]) T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_0_attention_query_key_value_q_weight2: T.Buffer((3072, 128), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale2: T.Buffer((3072, 32), "float16"), layer_norm49: T.Buffer((1, 1, 1024), "float16"), gpt_neox_layers_0_attention_query_key_value_bias2: T.Buffer((3072,), "float16"), T_add_intermediate_intermediate: T.Buffer((1, 1, 3072), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 3072), "float16", scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 3072), "float16", scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 3072), "float16", scope="local") gpt_neox_layers_0_attention_query_key_value_q_weight2_local = T.alloc_buffer((3072, 128), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(1536, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_attention_query_key_value_q_weight2_local"): v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(128, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, v1]) gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, v1] = gpt_neox_layers_0_attention_query_key_value_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], layer_norm49[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_attention_query_key_value_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + layer_norm49[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused] * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float16(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("T_add"): v0 = T.axis.spatial(3072, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_attention_query_key_value_bias2[v0]) T.writes(T_add_intermediate_intermediate[0, 0, v0]) T_add_intermediate_intermediate[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_attention_query_key_value_bias2[v0] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight3: T.Buffer((3072, 128), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale3: T.Buffer((3072, 32), "float16"), p_layer_norm98: T.handle, gpt_neox_layers_0_attention_query_key_value_bias3: T.Buffer((3072,), "float16"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm98 = T.match_buffer(p_layer_norm98, (1, seq_len, 1024), "float16") T_add_intermediate_intermediate = T.match_buffer(p_output0, (1, seq_len, 3072), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((3072, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 3072), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(768, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, layer_norm98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_query_key_value_bias3[v1]) T.writes(T_add_intermediate_intermediate[0, v0, v1]) T_add_intermediate_intermediate[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_query_key_value_bias3[v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((3072, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 3072), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(768, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, layer_norm98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_query_key_value_bias3[v1]) T.writes(T_add_intermediate_intermediate[0, v0, v1]) T_add_intermediate_intermediate[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_query_key_value_bias3[v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 3072), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(12, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(3072, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(3072, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm98[0, v1, v3], gpt_neox_layers_0_attention_query_key_value_q_weight3[v2, v3 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, layer_norm98[0, v1, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(3072, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_query_key_value_bias3[v2]) T.writes(T_add_intermediate_intermediate[0, v1, v2]) T_add_intermediate_intermediate[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_query_key_value_bias3[v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_0_attention_query_key_value_q_weight4: T.Buffer((3072, 128), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale4: T.Buffer((3072, 32), "float16"), p_layer_norm147: T.handle, gpt_neox_layers_0_attention_query_key_value_bias4: T.Buffer((3072,), "float16"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() layer_norm147 = T.match_buffer(p_layer_norm147, (batch_size, 1, 1024), "float16") T_add_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, 3072), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((3072, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 3072), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(768, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm147[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, layer_norm147[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_query_key_value_bias4[v1]) T.writes(T_add_intermediate_intermediate[v0, 0, v1]) T_add_intermediate_intermediate[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_query_key_value_bias4[v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((3072, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 3072), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 3072), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(768, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm147[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, layer_norm147[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(3072, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_query_key_value_bias4[v1]) T.writes(T_add_intermediate_intermediate[v0, 0, v1]) T_add_intermediate_intermediate[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_query_key_value_bias4[v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 3072), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(12, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(3072, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(3072, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm147[v1, 0, v3], gpt_neox_layers_0_attention_query_key_value_q_weight4[v2, v3 // 8], gpt_neox_layers_0_attention_query_key_value_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, layer_norm147[v1, 0, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(3072, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_query_key_value_bias4[v2]) T.writes(T_add_intermediate_intermediate[v1, 0, v2]) T_add_intermediate_intermediate[v1, 0, v2] = NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_query_key_value_bias4[v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_0_attention_dense_q_weight2: T.Buffer((1024, 128), "uint32"), gpt_neox_layers_0_attention_dense_q_scale2: T.Buffer((1024, 32), "float16"), lv293: T.Buffer((1, 1, 1024), "float16"), gpt_neox_layers_0_attention_dense_bias2: T.Buffer((1024,), "float16"), lv218: T.Buffer((1, 1, 1024), "float16"), input_embed: T.Buffer((1, 1, 1024), "float16"), T_add_intermediate_1_2_intermediate: T.Buffer((1, 1, 1024), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 1024), "float16", scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 1024), "float16", scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 1024), "float16", scope="local") gpt_neox_layers_0_attention_dense_q_weight2_local = T.alloc_buffer((1024, 128), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_attention_dense_q_weight2_local"): v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(128, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_attention_dense_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_attention_dense_q_weight2_local[v0, v1]) gpt_neox_layers_0_attention_dense_q_weight2_local[v0, v1] = gpt_neox_layers_0_attention_dense_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], lv293[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_attention_dense_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_attention_dense_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + lv293[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused] * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float16(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("T_add_2"): v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(lv218[0, 0, v0], NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_attention_dense_bias2[v0], input_embed[0, 0, v0]) T.writes(T_add_intermediate_1_2_intermediate[0, 0, v0]) T_add_intermediate_1_2_intermediate[0, 0, v0] = lv218[0, 0, v0] + (NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_attention_dense_bias2[v0]) + input_embed[0, 0, v0] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_0_attention_dense_q_weight4: T.Buffer((1024, 128), "uint32"), gpt_neox_layers_0_attention_dense_q_scale4: T.Buffer((1024, 32), "float16"), p_reshape291: T.handle, gpt_neox_layers_0_attention_dense_bias4: T.Buffer((1024,), "float16"), p_lv2: T.handle, p_input_embeds: T.handle, p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape291 = T.match_buffer(p_reshape291, (batch_size, 1, 1024), "float16") lv2 = T.match_buffer(p_lv2, (batch_size, 1, 1024), "float16") input_embeds = T.match_buffer(p_input_embeds, (batch_size, 1, 1024), "float16") T_add_intermediate_1_2_intermediate = T.match_buffer(p_output0, (batch_size, 1, 1024), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 1024), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], reshape291[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, reshape291[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(lv2[v0, 0, v1], NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_dense_bias4[v1], input_embeds[v0, 0, v1]) T.writes(T_add_intermediate_1_2_intermediate[v0, 0, v1]) T_add_intermediate_1_2_intermediate[v0, 0, v1] = lv2[v0, 0, v1] + (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_dense_bias4[v1]) + input_embeds[v0, 0, v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 1024), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], reshape291[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, reshape291[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(lv2[v0, 0, v1], NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_attention_dense_bias4[v1], input_embeds[v0, 0, v1]) T.writes(T_add_intermediate_1_2_intermediate[v0, 0, v1]) T_add_intermediate_1_2_intermediate[v0, 0, v1] = lv2[v0, 0, v1] + (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_attention_dense_bias4[v1]) + input_embeds[v0, 0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 1024), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(4, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], reshape291[v1, 0, v3], gpt_neox_layers_0_attention_dense_q_weight4[v2, v3 // 8], gpt_neox_layers_0_attention_dense_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, reshape291[v1, 0, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(lv2[v1, 0, v2], NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_dense_bias4[v2], input_embeds[v1, 0, v2]) T.writes(T_add_intermediate_1_2_intermediate[v1, 0, v2]) T_add_intermediate_1_2_intermediate[v1, 0, v2] = lv2[v1, 0, v2] + (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_dense_bias4[v2]) + input_embeds[v1, 0, v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight3: T.Buffer((1024, 128), "uint32"), gpt_neox_layers_0_attention_dense_q_scale3: T.Buffer((1024, 32), "float16"), p_reshape195: T.handle, gpt_neox_layers_0_attention_dense_bias3: T.Buffer((1024,), "float16"), p_lv74: T.handle, p_input_embeds: T.handle, p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape195 = T.match_buffer(p_reshape195, (1, seq_len, 1024), "float16") lv74 = T.match_buffer(p_lv74, (1, seq_len, 1024), "float16") input_embeds = T.match_buffer(p_input_embeds, (1, seq_len, 1024), "float16") T_add_intermediate_1_2_intermediate = T.match_buffer(p_output0, (1, seq_len, 1024), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 1024), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], reshape195[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, reshape195[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(lv74[0, v0, v1], NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_dense_bias3[v1], input_embeds[0, v0, v1]) T.writes(T_add_intermediate_1_2_intermediate[0, v0, v1]) T_add_intermediate_1_2_intermediate[0, v0, v1] = lv74[0, v0, v1] + (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_dense_bias3[v1]) + input_embeds[0, v0, v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 1024), "float16", scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 1024), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], reshape195[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, reshape195[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(lv74[0, v0, v1], NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_attention_dense_bias3[v1], input_embeds[0, v0, v1]) T.writes(T_add_intermediate_1_2_intermediate[0, v0, v1]) T_add_intermediate_1_2_intermediate[0, v0, v1] = lv74[0, v0, v1] + (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_attention_dense_bias3[v1]) + input_embeds[0, v0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 1024), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(4, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], reshape195[0, v1, v3], gpt_neox_layers_0_attention_dense_q_weight3[v2, v3 // 8], gpt_neox_layers_0_attention_dense_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, reshape195[0, v1, v3], T.float16(0.0)) * ((T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(lv74[0, v1, v2], NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_attention_dense_bias3[v2], input_embeds[0, v1, v2]) T.writes(T_add_intermediate_1_2_intermediate[0, v1, v2]) T_add_intermediate_1_2_intermediate[0, v1, v2] = lv74[0, v1, v2] + (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_attention_dense_bias3[v2]) + input_embeds[0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2: T.Buffer((4096, 128), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2: T.Buffer((4096, 32), "float16"), layer_norm50: T.Buffer((1, 1, 1024), "float16"), gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: T.Buffer((4096,), "float32"), compute_intermediate_intermediate: T.Buffer((1, 1, 4096), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 4096), scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 4096), scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 4096), scope="local") gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local = T.alloc_buffer((4096, 128), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(2048, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float32(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local"): v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(128, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, v1]) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, v1] = gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], layer_norm50[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + T.Cast("float32", layer_norm50[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused]) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float32(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("compute_1_1"): v0 = T.axis.spatial(4096, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v0]) T.writes(compute_intermediate_intermediate[0, 0, v0]) compute_intermediate_intermediate[0, 0, v0] = T.Cast("float16", (NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v0]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v0]) * T.float32(0.70710678118654757)) * T.float32(0.5))) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4: T.Buffer((4096, 128), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4: T.Buffer((4096, 32), "float16"), p_layer_norm148: T.handle, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: T.Buffer((4096,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() layer_norm148 = T.match_buffer(p_layer_norm148, (batch_size, 1, 1024), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, 4096), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((4096, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 4096), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 4096), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 4096), scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm148[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, layer_norm148[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((4096, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 4096), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 4096), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 4096), scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], layer_norm148[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, layer_norm148[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 4096), scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(16, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(4096, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(4096, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm148[v1, 0, v3], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < batch_size, layer_norm148[v1, 0, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(4096, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v2]) T.writes(compute_intermediate_intermediate[v1, 0, v2]) compute_intermediate_intermediate[v1, 0, v2] = T.Cast("float16", (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v2]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v2]) * T.float32(0.70710678118654757)) * T.float32(0.5))) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3: T.Buffer((4096, 128), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3: T.Buffer((4096, 32), "float16"), p_layer_norm99: T.handle, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: T.Buffer((4096,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm99 = T.match_buffer(p_layer_norm99, (1, seq_len, 1024), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, seq_len, 4096), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((4096, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 4096), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 4096), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 4096), scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm99[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, layer_norm99[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((4096, 1024), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 4096), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 4096), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 4096), scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(1024, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], layer_norm99[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, layer_norm99[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(4096, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", (NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v1]) * T.float32(0.70710678118654757)) * T.float32(0.5))) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 4096), scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(16, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(4096, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(4096, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm99[0, v1, v3], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < seq_len, layer_norm99[0, v1, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(4096, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v2]) T.writes(compute_intermediate_intermediate[0, v1, v2]) compute_intermediate_intermediate[0, v1, v2] = T.Cast("float16", (NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v2]) * (T.float32(0.5) + T.erf((NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v2]) * T.float32(0.70710678118654757)) * T.float32(0.5))) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2: T.Buffer((1024, 512), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2: T.Buffer((1024, 128), "float16"), lv294: T.Buffer((1, 1, 4096), "float16"), gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: T.Buffer((1024,), "float32"), compute_intermediate_intermediate: T.Buffer((1, 1, 1024), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, 1024), scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, 1024), scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, 1024), scope="local") gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local = T.alloc_buffer((1024, 512), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float32(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local"): v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(512, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2[v0, v1]) T.writes(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, v1]) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, v1] = gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], lv294[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + T.Cast("float32", lv294[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused]) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float32(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("compute_1"): v0 = T.axis.spatial(1024, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.reads(NT_matmul_intermediate_local[0, 0, v0], gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[v0]) T.writes(compute_intermediate_intermediate[0, 0, v0]) compute_intermediate_intermediate[0, 0, v0] = T.Cast("float16", NT_matmul_intermediate_local[0, 0, v0] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[v0]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4: T.Buffer((1024, 512), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4: T.Buffer((1024, 128), "float16"), p_lv1: T.handle, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: T.Buffer((1024,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv1 = T.match_buffer(p_lv1, (batch_size, 1, 4096), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, 1024), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 4096), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 1024), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 1024), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 1024), scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(4096, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 4096), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 1024), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 1024), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 1024), scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(4096, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.Cast("float32", T.if_then_else(v0 < batch_size, lv1[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_intermediate_pad_local[v0, 0, v1]) with T.init(): NT_matmul_intermediate_pad_local[v0, 0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[v0, 0, v1] = NT_matmul_intermediate_pad_local[v0, 0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_intermediate_pad_local[v0, 0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) T.writes(compute_intermediate_intermediate[v0, 0, v1]) compute_intermediate_intermediate[v0, 0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[v0, 0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v1]) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 1024), scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(4, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(256, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(4096, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], lv1[v1, 0, v3], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < batch_size, lv1[v1, 0, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v2]) T.writes(compute_intermediate_intermediate[v1, 0, v2]) compute_intermediate_intermediate[v1, 0, v2] = T.Cast("float16", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3: T.Buffer((1024, 512), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3: T.Buffer((1024, 128), "float16"), p_lv98: T.handle, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: T.Buffer((1024,), "float32"), p_output0: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv98 = T.match_buffer(p_lv98, (1, seq_len, 4096), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, seq_len, 1024), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 4096), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 1024), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 1024), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 1024), scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(4096, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], lv98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, lv98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() dequantize_intermediate_local = T.alloc_buffer((1024, 4096), "float16", scope="local") NT_matmul_intermediate_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 1024), scope="local") NT_matmul_intermediate_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 1024), scope="local") NT_matmul_intermediate_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 1024), scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(256, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float32(0.0) for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_0_1, ax1 in T.grid(2, 8): for ax0_1 in T.vectorized(1): with T.block("dequantize"): v0 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax0_0_1 + ax0_1) v1 = T.axis.spatial(4096, ax2_fused_0 * 256 + ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax1) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32]) T.writes(dequantize_intermediate_local[v0, v1]) dequantize_intermediate_local[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v0, v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v0, v1 // 32] for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], lv98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.Cast("float32", T.if_then_else(v0 < seq_len, lv98[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0))) * T.Cast("float32", dequantize_intermediate_local[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_intermediate_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_intermediate_pad_local[0, v0, v1]) with T.init(): NT_matmul_intermediate_pad_local[0, v0, v1] = T.float32(0.0) NT_matmul_intermediate_pad_local[0, v0, v1] = NT_matmul_intermediate_pad_local[0, v0, v1] + NT_matmul_intermediate_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_intermediate_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(1024, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_intermediate_pad_local[0, v0, v1], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) T.writes(compute_intermediate_intermediate[0, v0, v1]) compute_intermediate_intermediate[0, v0, v1] = T.Cast("float16", NT_matmul_intermediate_pad_local[0, v0, v1] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v1]) else: with T.block("root"): T.reads() T.writes() NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 1024), scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(4, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(256, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(4096, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], lv98[0, v1, v3], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v2, v3 // 8], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < seq_len, lv98[0, v1, v3], T.float16(0.0))) * T.Cast("float32", (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v2, v3 // 32]) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(1024, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v2]) T.writes(compute_intermediate_intermediate[0, v1, v2]) compute_intermediate_intermediate[0, v1, v2] = T.Cast("float16", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8(p_embed_out_q_weight2: T.handle, p_embed_out_q_scale2: T.handle, layer_norm97: T.Buffer((1, 1, 1024), "float16"), p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() embed_out_q_weight2 = T.match_buffer(p_embed_out_q_weight2, (vocab_size, 128), "uint32") embed_out_q_scale2 = T.match_buffer(p_embed_out_q_scale2, (vocab_size, 32), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, 1, vocab_size)) # with T.block("root"): NT_matmul_intermediate_local = T.alloc_buffer((1, 1, vocab_size), "float16", scope="local") NT_matmul_intermediate_rf_local = T.alloc_buffer((256, 1, 1, vocab_size), "float16", scope="local") NT_matmul_intermediate_rf_local_1 = T.alloc_buffer((32, 1, 1, vocab_size), "float16", scope="local") embed_out_q_weight2_local = T.alloc_buffer((vocab_size, 128), "uint32", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding((vocab_size + 1) // 2, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1_init) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init < vocab_size) T.reads() T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_0_fused_ax1_1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(1): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("embed_out_q_weight2_local"): v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(128, ax1_0_fused_ax1_1_fused_0 * 32 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 < vocab_size) T.reads(embed_out_q_weight2[v0, v1]) T.writes(embed_out_q_weight2_local[v0, v1]) embed_out_q_weight2_local[v0, v1] = embed_out_q_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_0_fused_ax1_1_fused_2 in T.grid(1, 1): for ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused = T.axis.spatial(256, ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + ax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_0_fused_ax1_1_fused_2, vax1_0_fused_ax1_1_fused_0 = T.axis.remap("RR", [ax1_0_fused_ax1_1_fused_2, ax1_0_fused_ax1_1_fused_0]) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2 < vocab_size) T.reads(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0], layer_norm97[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused], embed_out_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], embed_out_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) T.writes(NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0]) NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] = NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused, 0, 0, v0] + layer_norm97[0, 0, vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused] * ((T.Cast("float16", T.bitwise_and(T.shift_right(embed_out_q_weight2_local[v0, vax1_0_fused_ax1_1_fused_0 * 32 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused // 8 + vax1_0_fused_ax1_1_fused_2], T.Cast("uint32", (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * embed_out_q_scale2[v0, (vax1_0_fused_ax1_1_fused_2 * 8 + vax1_0_fused_ax1_1_fused_0 * 256 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused) // 32]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax2_fused_0_ax2_fused_1_fused % 2 + (ax2_fused_2_0 + ax2_fused_2_1)) < vocab_size) T.reads() T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax2_fused_0_ax2_fused_1_fused % 2 + (ax2_fused_2_0 + ax2_fused_2_1)) < vocab_size) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0], NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] = NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] + NT_matmul_intermediate_rf_local[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 * 8 + vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax1_fused_0_ax1_fused_1_fused % 2 + ax1_fused_2) < vocab_size) T.reads(NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul_intermediate_local[0, 0, v0]) with T.init(): NT_matmul_intermediate_local[0, 0, v0] = T.float16(0.0) NT_matmul_intermediate_local[0, 0, v0] = NT_matmul_intermediate_local[0, 0, v0] + NT_matmul_intermediate_rf_local_1[vax1_0_fused_ax1_1_fused_1_ax1_0_fused_ax1_1_fused_3_fused_0, 0, 0, v0] for ax0_fused_0_ax0_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0_fused_2 in range(1): with T.block("compute_1"): v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax0_fused_0_ax0_fused_1_fused + ax0_fused_2) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax0_fused_0_ax0_fused_1_fused % 2 + ax0_fused_2) < vocab_size) T.reads(NT_matmul_intermediate_local[0, 0, v0]) T.writes(compute_intermediate_intermediate[0, 0, v0]) compute_intermediate_intermediate[0, 0, v0] = T.Cast("float32", NT_matmul_intermediate_local[0, 0, v0]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2(p_embed_out_q_weight4: T.handle, p_embed_out_q_scale4: T.handle, p_layer_norm195: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() embed_out_q_weight4 = T.match_buffer(p_embed_out_q_weight4, (vocab_size, 128), "uint32") embed_out_q_scale4 = T.match_buffer(p_embed_out_q_scale4, (vocab_size, 32), "float16") batch_size = T.int32() layer_norm195 = T.match_buffer(p_layer_norm195, (batch_size, 1, 1024), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (batch_size, 1, vocab_size)) # with T.block("root"): NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, (vocab_size + 255) // 256 * 256), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding((vocab_size + 255) // 256, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], layer_norm195[v1, 0, v3], embed_out_q_weight4[v2, v3 // 8], embed_out_q_scale4[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, layer_norm195[v1, 0, v3], T.float16(0.0)) * T.if_then_else(v2 < vocab_size, (T.Cast("float16", T.bitwise_and(T.shift_right(embed_out_q_weight4[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * embed_out_q_scale4[v2, v3 // 32], T.float16(0.0)) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size and ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1 < vocab_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) T.writes(compute_intermediate_intermediate[v1, 0, v2]) compute_intermediate_intermediate[v1, 0, v2] = T.Cast("float32", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5(p_embed_out_q_weight3: T.handle, p_embed_out_q_scale3: T.handle, p_take1: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() embed_out_q_weight3 = T.match_buffer(p_embed_out_q_weight3, (vocab_size, 128), "uint32") embed_out_q_scale3 = T.match_buffer(p_embed_out_q_scale3, (vocab_size, 32), "float16") batch_size = T.int32() take1 = T.match_buffer(p_take1, (1, batch_size, 1024), "float16") compute_intermediate_intermediate = T.match_buffer(p_output0, (1, batch_size, vocab_size)) # with T.block("root"): NT_matmul_intermediate_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, (vocab_size + 255) // 256 * 256), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding((vocab_size + 255) // 256, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(64, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(1024, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_intermediate_reindex_pad_local[0, v1, v2], take1[0, v1, v3], embed_out_q_weight3[v2, v3 // 8], embed_out_q_scale3[v2, v3 // 32]) T.writes(NT_matmul_intermediate_reindex_pad_local[0, v1, v2]) NT_matmul_intermediate_reindex_pad_local[0, v1, v2] = NT_matmul_intermediate_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, take1[0, v1, v3], T.float16(0.0)) * T.if_then_else(v2 < vocab_size, (T.Cast("float16", T.bitwise_and(T.shift_right(embed_out_q_weight3[v2, v3 // 8], T.Cast("uint32", v3 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * embed_out_q_scale3[v2, v3 // 32], T.float16(0.0)) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_intermediate_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size and ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1 < vocab_size) T.reads(NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) T.writes(compute_intermediate_intermediate[0, v1, v2]) compute_intermediate_intermediate[0, v1, v2] = T.Cast("float32", NT_matmul_intermediate_reindex_pad_local[v0, v1, v2]) @T.prim_func def gpt_neox_q4f16_1_fused_dequantize_take1(p_gpt_neox_embed_in_q_weight: T.handle, p_gpt_neox_embed_in_q_scale: T.handle, p_input_ids: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() gpt_neox_embed_in_q_weight = T.match_buffer(p_gpt_neox_embed_in_q_weight, (vocab_size, 128), "uint32") gpt_neox_embed_in_q_scale = T.match_buffer(p_gpt_neox_embed_in_q_scale, (vocab_size, 32), "float16") seq_len = T.int32() input_ids = T.match_buffer(p_input_ids, (seq_len,), "int32") T_take_intermediate = T.match_buffer(p_output0, (seq_len, 1024), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 4, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_take"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 1024) v1 = T.axis.spatial(1024, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 1024) T.reads(input_ids[v0], gpt_neox_embed_in_q_weight[input_ids[v0], v1 // 8], gpt_neox_embed_in_q_scale[input_ids[v0], v1 // 32]) T.writes(T_take_intermediate[v0, v1]) T_take_intermediate[v0, v1] = (T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_embed_in_q_weight[input_ids[v0], v1 // 8], T.Cast("uint32", v1 % 8 * 4)), T.uint32(15))) - T.float16(7.0)) * gpt_neox_embed_in_q_scale[input_ids[v0], v1 // 32] @T.prim_func def gpt_neox_q4f16_1_fused_reshape10_reshape11(lv124: T.Buffer((1, 16, 64), "float16"), T_reshape_intermediate_1: T.Buffer((1, 1, 1024), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_fused_0 in T.thread_binding(4, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape_1"): v0 = T.axis.spatial(1024, ax0_fused_0 * 256 + ax0_fused_1) T.reads(lv124[0, v0 // 64, v0 % 64]) T.writes(T_reshape_intermediate_1[0, 0, v0]) T_reshape_intermediate_1[0, 0, v0] = lv124[0, v0 // 64, v0 % 64] @T.prim_func def gpt_neox_q4f16_1_fused_reshape8_reshape9(add144: T.Buffer((1, 1, 3072), "float16"), T_reshape_intermediate_1: T.Buffer((1, 48, 64), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(12, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape_1"): v0 = T.axis.spatial(48, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 64) v1 = T.axis.spatial(64, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 64) T.reads(add144[0, 0, v0 * 64 + v1]) T.writes(T_reshape_intermediate_1[0, v0, v1]) T_reshape_intermediate_1[0, v0, v1] = add144[0, 0, v0 * 64 + v1] @T.prim_func def gpt_neox_q4f16_1_fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() qkv = T.match_buffer(var_qkv, (seq_len, 48, 64), "float16") position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) q = T.match_buffer(var_q, (seq_len, 16, 64), "float16") k = T.match_buffer(var_k, (seq_len, 16, 64), "float16") v = T.match_buffer(var_v, (seq_len, 16, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 12, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("llama_fused_rope"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 3072) v1 = T.axis.spatial(48, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 3072 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(position_map[v0], qkv[v0, v1, v2 + -8:v2 + -8 + 17]) T.writes(q[v0, v1, v2], k[v0, v1 + -16, v2], v[v0, v1 + -32, v2]) if v1 < 16: freq = T.float32() q[v0, v1, v2] = T.if_then_else(apply_rope > 0 and v2 < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[v0, v1, v2]) + T.sin(freq) * T.Cast("float32", T.if_then_else(v2 < 8, qkv[v0, v1, v2 + 8] * T.float16(-1.0), qkv[v0, v1, v2 + -8]))), where={freq: T.Cast("float32", position_map[v0]) / T.pow(T.float32(10000.0), T.Cast("float32", v2 * 2 % 16) / T.float32(16.0))}), qkv[v0, v1, v2]) else: if v1 < 32: freq = T.float32() k[v0, v1 + -16, v2] = T.if_then_else(apply_rope > 0 and v2 < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[v0, v1, v2]) + T.sin(freq) * T.Cast("float32", T.if_then_else(v2 < 8, qkv[v0, v1, v2 + 8] * T.float16(-1.0), qkv[v0, v1, v2 + -8]))), where={freq: T.Cast("float32", position_map[v0]) / T.pow(T.float32(10000.0), T.Cast("float32", v2 * 2 % 16) / T.float32(16.0))}), qkv[v0, v1, v2]) else: v[v0, v1 + -32, v2] = qkv[v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (m, n)) batch_size = T.int32(is_size_var=True) indices = T.match_buffer(var_indices, (batch_size,), "int32") dst = T.match_buffer(var_dst, (batch_size, n)) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding((batch_size * n + 255) // 256, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("gather_2d"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % (n * batch_size) // n) v1 = T.axis.spatial(n, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % n) T.where(ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1 < batch_size * n) T.reads(src[indices[v0], v1], indices[v0]) T.writes(dst[v0, v1]) dst[v0, v1] = src[indices[v0], v1] @T.prim_func def gpt_neox_q4f16_1_index(var_layer_norm48: T.handle, index: T.Buffer((1, 1, 1024), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm48 = T.match_buffer(var_layer_norm48, (1, seq_len, 1024), "float16") # with T.block("root"): for ax0_fused_0 in T.thread_binding(4, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("index"): v0 = T.axis.spatial(1024, ax0_fused_0 * 256 + ax0_fused_1) T.reads(layer_norm48[0, seq_len - 1, v0]) T.writes(index[0, 0, v0]) index[0, 0, v0] = layer_norm48[0, seq_len - 1, v0] @T.prim_func def gpt_neox_q4f16_1_layer_norm(var_input_embeds: T.handle, gpt_neox_layers_0_input_layernorm_weight4: T.Buffer((1024,), "float16"), gpt_neox_layers_0_input_layernorm_bias4: T.Buffer((1024,), "float16"), var_T_layer_norm: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() input_embeds = T.match_buffer(var_input_embeds, (batch_size, 1, 1024), "float16") T_layer_norm = T.match_buffer(var_T_layer_norm, (batch_size, 1, 1024), "float16") # with T.block("root"): input_embeds_red_temp_v0_shared = T.alloc_buffer((batch_size, 1), scope="shared") input_embeds_red_temp_v1_shared = T.alloc_buffer((batch_size, 1), scope="shared") for ax0_fused in T.thread_binding(batch_size, thread="blockIdx.x"): for ax0 in range(1): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("input_embeds_red_temp"): v0 = T.axis.spatial(batch_size, ax0_fused + ax0) v1 = T.axis.reduce(1024, ax1_fused_0 * 256 + ax1_fused_1) T.reads(input_embeds[v0, 0, v1]) T.writes(input_embeds_red_temp_v0_shared[v0, 0], input_embeds_red_temp_v1_shared[v0, 0]) with T.init(): input_embeds_red_temp_v0_shared[v0, 0] = T.float32(0.0) input_embeds_red_temp_v1_shared[v0, 0] = T.float32(0.0) v_input_embeds_red_temp_v0: T.float32 = input_embeds_red_temp_v0_shared[v0, 0] + T.Cast("float32", input_embeds[v0, 0, v1]) v_input_embeds_red_temp_v1: T.float32 = input_embeds_red_temp_v1_shared[v0, 0] + T.Cast("float32", input_embeds[v0, 0, v1]) * T.Cast("float32", input_embeds[v0, 0, v1]) input_embeds_red_temp_v0_shared[v0, 0] = v_input_embeds_red_temp_v0 input_embeds_red_temp_v1_shared[v0, 0] = v_input_embeds_red_temp_v1 for ax1_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("T_layer_norm"): v0 = T.axis.spatial(batch_size, ax0_fused) v1 = T.axis.spatial(1024, ax1_0 * 256 + ax1_1) T.reads(input_embeds[v0, 0, v1], input_embeds_red_temp_v0_shared[v0, 0], input_embeds_red_temp_v1_shared[v0, 0], gpt_neox_layers_0_input_layernorm_weight4[v1], gpt_neox_layers_0_input_layernorm_bias4[v1]) T.writes(T_layer_norm[v0, 0, v1]) T_layer_norm[v0, 0, v1] = T.Cast("float16", (T.Cast("float32", input_embeds[v0, 0, v1]) - input_embeds_red_temp_v0_shared[v0, 0] * T.float32(0.0009765625)) * T.rsqrt(input_embeds_red_temp_v1_shared[v0, 0] * T.float32(0.0009765625) - input_embeds_red_temp_v0_shared[v0, 0] * T.float32(0.0009765625) * (input_embeds_red_temp_v0_shared[v0, 0] * T.float32(0.0009765625)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight4[v1] + gpt_neox_layers_0_input_layernorm_bias4[v1] @T.prim_func def gpt_neox_q4f16_1_layer_norm1(var_input_embeds: T.handle, gpt_neox_layers_0_input_layernorm_weight3: T.Buffer((1024,), "float16"), gpt_neox_layers_0_input_layernorm_bias3: T.Buffer((1024,), "float16"), var_T_layer_norm: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() input_embeds = T.match_buffer(var_input_embeds, (1, seq_len, 1024), "float16") T_layer_norm = T.match_buffer(var_T_layer_norm, (1, seq_len, 1024), "float16") # with T.block("root"): input_embeds_red_temp_v0_shared = T.alloc_buffer((1, seq_len), scope="shared") input_embeds_red_temp_v1_shared = T.alloc_buffer((1, seq_len), scope="shared") for ax0_fused in T.thread_binding(seq_len, thread="blockIdx.x"): for ax0 in range(1): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("input_embeds_red_temp"): v0 = T.axis.spatial(seq_len, ax0_fused + ax0) v1 = T.axis.reduce(1024, ax1_fused_0 * 256 + ax1_fused_1) T.reads(input_embeds[0, v0, v1]) T.writes(input_embeds_red_temp_v0_shared[0, v0], input_embeds_red_temp_v1_shared[0, v0]) with T.init(): input_embeds_red_temp_v0_shared[0, v0] = T.float32(0.0) input_embeds_red_temp_v1_shared[0, v0] = T.float32(0.0) v_input_embeds_red_temp_v0: T.float32 = input_embeds_red_temp_v0_shared[0, v0] + T.Cast("float32", input_embeds[0, v0, v1]) v_input_embeds_red_temp_v1: T.float32 = input_embeds_red_temp_v1_shared[0, v0] + T.Cast("float32", input_embeds[0, v0, v1]) * T.Cast("float32", input_embeds[0, v0, v1]) input_embeds_red_temp_v0_shared[0, v0] = v_input_embeds_red_temp_v0 input_embeds_red_temp_v1_shared[0, v0] = v_input_embeds_red_temp_v1 for ax1_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("T_layer_norm"): v0 = T.axis.spatial(seq_len, ax0_fused) v1 = T.axis.spatial(1024, ax1_0 * 256 + ax1_1) T.reads(input_embeds[0, v0, v1], input_embeds_red_temp_v0_shared[0, v0], input_embeds_red_temp_v1_shared[0, v0], gpt_neox_layers_0_input_layernorm_weight3[v1], gpt_neox_layers_0_input_layernorm_bias3[v1]) T.writes(T_layer_norm[0, v0, v1]) T_layer_norm[0, v0, v1] = T.Cast("float16", (T.Cast("float32", input_embeds[0, v0, v1]) - input_embeds_red_temp_v0_shared[0, v0] * T.float32(0.0009765625)) * T.rsqrt(input_embeds_red_temp_v1_shared[0, v0] * T.float32(0.0009765625) - input_embeds_red_temp_v0_shared[0, v0] * T.float32(0.0009765625) * (input_embeds_red_temp_v0_shared[0, v0] * T.float32(0.0009765625)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight3[v1] + gpt_neox_layers_0_input_layernorm_bias3[v1] @T.prim_func def gpt_neox_q4f16_1_layer_norm2(input_embed: T.Buffer((1, 1, 1024), "float16"), gpt_neox_layers_0_input_layernorm_weight2: T.Buffer((1024,), "float16"), gpt_neox_layers_0_input_layernorm_bias2: T.Buffer((1024,), "float16"), T_layer_norm: T.Buffer((1, 1, 1024), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): input_embed_red_temp_v0_shared = T.alloc_buffer((1, 1), scope="shared") input_embed_red_temp_v1_shared = T.alloc_buffer((1, 1), scope="shared") for ax0_fused in T.thread_binding(1, thread="blockIdx.x"): for ax0 in range(1): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_fused_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("input_embed_red_temp"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.reduce(1024, ax1_fused_0 * 256 + ax1_fused_1) T.reads(input_embed[0, 0, v1]) T.writes(input_embed_red_temp_v0_shared[0, 0], input_embed_red_temp_v1_shared[0, 0]) with T.init(): input_embed_red_temp_v0_shared[0, 0] = T.float32(0.0) input_embed_red_temp_v1_shared[0, 0] = T.float32(0.0) v_input_embed_red_temp_v0: T.float32 = input_embed_red_temp_v0_shared[0, 0] + T.Cast("float32", input_embed[0, 0, v1]) v_input_embed_red_temp_v1: T.float32 = input_embed_red_temp_v1_shared[0, 0] + T.Cast("float32", input_embed[0, 0, v1]) * T.Cast("float32", input_embed[0, 0, v1]) input_embed_red_temp_v0_shared[0, 0] = v_input_embed_red_temp_v0 input_embed_red_temp_v1_shared[0, 0] = v_input_embed_red_temp_v1 for ax1_1 in T.thread_binding(256, thread="threadIdx.x"): for ax1_0 in T.serial(4, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("T_layer_norm"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial(1024, ax1_0 * 256 + ax1_1) T.reads(input_embed[0, 0, v1], input_embed_red_temp_v0_shared[0, 0], input_embed_red_temp_v1_shared[0, 0], gpt_neox_layers_0_input_layernorm_weight2[v1], gpt_neox_layers_0_input_layernorm_bias2[v1]) T.writes(T_layer_norm[0, 0, v1]) T_layer_norm[0, 0, v1] = T.Cast("float16", (T.Cast("float32", input_embed[0, 0, v1]) - input_embed_red_temp_v0_shared[0, 0] * T.float32(0.0009765625)) * T.rsqrt(input_embed_red_temp_v1_shared[0, 0] * T.float32(0.0009765625) - input_embed_red_temp_v0_shared[0, 0] * T.float32(0.0009765625) * (input_embed_red_temp_v0_shared[0, 0] * T.float32(0.0009765625)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight2[v1] + gpt_neox_layers_0_input_layernorm_bias2[v1] @T.prim_func def gpt_neox_q4f16_1_merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) V = T.match_buffer(v, (N, H, D), "float16") S = T.match_buffer(s, (N, H)) V_other = T.match_buffer(v_other, (N, H, D), "float16") S_other = T.match_buffer(s_other, (N, H)) # with T.block("root"): for bx in T.thread_binding(N, thread="blockIdx.x"): for by in T.thread_binding(1, thread="blockIdx.y"): for ty in T.thread_binding(16, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): with T.block("merge"): T.reads(S[bx, ty + by * 16], S_other[bx, ty + by * 16], V[bx, ty + by * 16, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 16, tx * 4:tx * 4 + 4]) T.writes(V[bx, ty + by * 16, tx * 4:tx * 4 + 4], S[bx, ty + by * 16]) s_val = T.alloc_buffer((1,), scope="local") s_other_val = T.alloc_buffer((1,), scope="local") s_max = T.alloc_buffer((1,), scope="local") scale = T.alloc_buffer((1,), scope="local") other_scale = T.alloc_buffer((1,), scope="local") v_vec = T.alloc_buffer((4,), "float16", scope="local") v_other_vec = T.alloc_buffer((4,), "float16", scope="local") s_val[0] = S[bx, ty + by * 16] s_other_val[0] = S_other[bx, ty + by * 16] s_max[0] = T.max(s_val[0], s_other_val[0]) s_val[0] = T.exp2(s_val[0] - s_max[0]) s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) for vec in T.vectorized(4): v_vec[vec] = V[bx, ty + by * 16, tx * 4 + vec] for vec in T.vectorized(4): v_other_vec[vec] = V_other[bx, ty + by * 16, tx * 4 + vec] for vec in range(4): v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) for vec in T.vectorized(4): V[bx, ty + by * 16, tx * 4 + vec] = v_vec[vec] S[bx, ty + by * 16] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] @T.prim_func def gpt_neox_q4f16_1_reshape(var_add432: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() add432 = T.match_buffer(var_add432, (batch_size, 1, 3072), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 48, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 12, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 3072) v1 = T.axis.spatial(48, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 3072 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(add432[v0, 0, v1 * 64 + v2]) T.writes(T_reshape[v0, 0, v1, v2]) T_reshape[v0, 0, v1, v2] = add432[v0, 0, v1 * 64 + v2] @T.prim_func def gpt_neox_q4f16_1_reshape1(var_reshape288: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape288 = T.match_buffer(var_reshape288, (batch_size, 1, 48, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 48, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 12, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 3072) v1 = T.axis.spatial(48, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 3072 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(reshape288[v0, 0, v1, v2]) T.writes(T_reshape[v0, v1, v2]) T_reshape[v0, v1, v2] = reshape288[v0, 0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape2(var_lv366: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv366 = T.match_buffer(var_lv366, (batch_size, 16, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 16, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 4, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1024) v1 = T.axis.spatial(16, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1024 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(lv366[v0, v1, v2]) T.writes(T_reshape[v0, 0, v1, v2]) T_reshape[v0, 0, v1, v2] = lv366[v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape3(var_reshape290: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape290 = T.match_buffer(var_reshape290, (batch_size, 1, 16, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 1024), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 4, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 1024) v1 = T.axis.spatial(1024, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 1024) T.reads(reshape290[v0, 0, v1 // 64, v1 % 64]) T.writes(T_reshape[v0, 0, v1]) T_reshape[v0, 0, v1] = reshape290[v0, 0, v1 // 64, v1 % 64] @T.prim_func def gpt_neox_q4f16_1_reshape4(var_add288: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() add288 = T.match_buffer(var_add288, (1, seq_len, 3072), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 48, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 12, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 3072) v1 = T.axis.spatial(48, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 3072 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(add288[0, v0, v1 * 64 + v2]) T.writes(T_reshape[0, v0, v1, v2]) T_reshape[0, v0, v1, v2] = add288[0, v0, v1 * 64 + v2] @T.prim_func def gpt_neox_q4f16_1_reshape5(var_reshape192: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape192 = T.match_buffer(var_reshape192, (1, seq_len, 48, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (seq_len, 48, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 12, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 3072) v1 = T.axis.spatial(48, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 3072 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(reshape192[0, v0, v1, v2]) T.writes(T_reshape[v0, v1, v2]) T_reshape[v0, v1, v2] = reshape192[0, v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape6(var_lv245: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv245 = T.match_buffer(var_lv245, (seq_len, 16, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 16, 64), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 4, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 1024) v1 = T.axis.spatial(16, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 1024 // 64) v2 = T.axis.spatial(64, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 64) T.reads(lv245[v0, v1, v2]) T.writes(T_reshape[0, v0, v1, v2]) T_reshape[0, v0, v1, v2] = lv245[v0, v1, v2] @T.prim_func def gpt_neox_q4f16_1_reshape7(var_reshape194: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape194 = T.match_buffer(var_reshape194, (1, seq_len, 16, 64), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 1024), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 4, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 1024) v1 = T.axis.spatial(1024, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 1024) T.reads(reshape194[0, v0, v1 // 64, v1 % 64]) T.writes(T_reshape[0, v0, v1]) T_reshape[0, v0, v1] = reshape194[0, v0, v1 // 64, v1 % 64] @T.prim_func def gpt_neox_q4f16_1_scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (batch_size, n)) indices = T.match_buffer(var_indices, (batch_size,), "int32") m = T.int32(is_size_var=True) dst = T.match_buffer(var_dst, (m, n)) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding((batch_size * n + 255) // 256, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("scatter_2d"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % (n * batch_size) // n) v1 = T.axis.spatial(n, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % n) T.where(ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1 < batch_size * n) T.reads(src[v0, v1], indices[v0]) T.writes(dst[indices[v0], v1]) dst[indices[v0], v1] = src[v0, v1] @T.prim_func def gpt_neox_q4f16_1_shape_func(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = H[T.int64(1)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func1(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = T.int64(128) * H[T.int64(1)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func2(H: T.Buffer((T.int64(3),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(2)] = H[T.int64(0)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func3(H: T.Buffer((T.int64(5),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(4)] = T.int64(512) * H[T.int64(1)] H[T.int64(3)] = (H[T.int64(1)] + T.int64(4096) - T.int64(1)) // T.int64(4096) H[T.int64(2)] = T.int64(512) * ((H[T.int64(1)] + T.int64(4096) - T.int64(1)) // T.int64(4096)) @T.prim_func def gpt_neox_q4f16_1_shape_func4(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = T.int64(2048) * H[T.int64(1)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_shape_func5(H: T.Buffer((T.int64(5),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(4)] = T.int64(128) * H[T.int64(2)] * T.int64(4) @T.prim_func def gpt_neox_q4f16_1_softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int32(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0_1 in T.thread_binding(32, thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + 31) // 32, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * 32 + ax0_1) T.where(ax0_0 * 32 + ax0_1 < num_chunks) T.reads(chunked_max[v0, v1]) T.writes(temp_max_shared[v0]) with T.init(): temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) for ax0_1 in T.thread_binding(32, thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + 31) // 32, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * 32 + ax0_1) T.where(ax0_0 * 32 + ax0_1 < num_chunks) T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) T.writes(temp_sum_shared[v0]) with T.init(): temp_sum_shared[v0] = T.float32(0.0) temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) for l2_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): for l2_1 in T.thread_binding(8, thread="threadIdx.y"): for l2_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("log_pad"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) v2 = T.axis.spatial(4096, l2_0 * 256 + l2_1 * 32 + l2_2) T.reads(temperature[v0], A[v0, v1 * 4096 + v2], temp_sum_shared[v0], temp_max_shared[v0]) T.writes(softmax[v0, v1 * 4096 + v2]) if v1 * 4096 + v2 < vocab_size: softmax[v0, v1 * 4096 + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * 4096 + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * 4096 + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) @T.prim_func def gpt_neox_q4f16_1_take(var_layer_norm146: T.handle, var_logit_positions: T.handle, var_T_take: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() layer_norm146 = T.match_buffer(var_layer_norm146, (1, seq_len, 1024), "float16") batch_size = T.int32() logit_positions = T.match_buffer(var_logit_positions, (batch_size,), "int32") T_take = T.match_buffer(var_T_take, (1, batch_size, 1024), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 4, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_take"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 1024) v1 = T.axis.spatial(1024, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 1024) T.reads(layer_norm146[0, logit_positions[v0], v1], logit_positions[v0]) T.writes(T_take[0, v0, v1]) T_take[0, v0, v1] = layer_norm146[0, logit_positions[v0], v1] @T.prim_func def gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) num_pages, page_size = T.int32(), T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (num_pages, 2, 16, page_size, 64), "float16", offset_factor=1) seqlen = T.int32(is_size_var=True) position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) k_data = T.match_buffer(var_k_data, (24, seqlen, 16, 64), "float16") v_data = T.match_buffer(var_v_data, (24, seqlen, 16, 64), "float16") # with T.block("root"): for p_h_d_fused_0 in T.thread_binding(seqlen * 4, thread="blockIdx.x"): for p_h_d_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy0"): vp = T.axis.spatial(seqlen, (p_h_d_fused_0 * 256 + p_h_d_fused_1) // 1024) vh = T.axis.spatial(16, (p_h_d_fused_0 * 256 + p_h_d_fused_1) % 1024 // 64) vd = T.axis.spatial(64, (p_h_d_fused_0 * 256 + p_h_d_fused_1) % 64) T.reads(position_map[vp], pages[position_map[vp] // page_size, 0:2, vh, position_map[vp] % page_size, vd]) T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) position: T.int32 = position_map[vp] k_data[layer_id, vp, vh, vd] = pages[position // page_size, 0, vh, position % page_size, vd] v_data[layer_id, vp, vh, vd] = pages[position // page_size, 1, vh, position % page_size, vd] @T.prim_func def gpt_neox_q4f16_1_tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 16, 16, 64), "float16", offset_factor=1) ntoken = T.int32(is_size_var=True) k_data = T.match_buffer(var_k_data, (ntoken, 16, 64), "float16") v_data = T.match_buffer(var_v_data, (ntoken, 16, 64), "float16") position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) # with T.block("root"): for global_pos_h_f_fused_0 in T.thread_binding(ntoken * 4, thread="blockIdx.x"): for global_pos_h_f_fused_1 in T.thread_binding(256, thread="threadIdx.x"): if position_map[(global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 1024] != -1: with T.block("k_transpose_append"): vgpos = T.axis.spatial(ntoken, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 1024) vh = T.axis.spatial(16, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 1024 // 64) vf = T.axis.spatial(64, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 64) T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] with T.block("v_transpose_append"): vgpos = T.axis.spatial(ntoken, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 1024) vh = T.axis.spatial(16, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 1024 // 64) vf = T.axis.spatial(64, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 64) T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] @T.prim_func def gpt_neox_q4f16_1_tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 16, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 16, 32, 64), "float16") page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 16, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 16)) tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) total_tree_order_len = T.int32(is_size_var=True) tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) # with T.block("root"): assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(16, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((32, 64), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 K_smem[i, j] = pages[page_no, 0, by, page_offset, j] else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @R.function def _metadata() -> R.Object: shape_heap: R.Object = R.null_value() return R.str("{\"model_type\": \"gpt_neox\", \"quantization\": \"q4f16_1\", \"context_window_size\": 2048, \"sliding_window_size\": -1, \"attention_sink_size\": -1, \"prefill_chunk_size\": 2048, \"tensor_parallel_shards\": 1, \"pipeline_parallel_stages\": 1, \"disaggregation\": false, \"kv_state_kind\": \"kv_cache\", \"max_batch_size\": 128, \"params\": [{\"name\": \"gpt_neox.embed_in.q_weight\", \"shape\": [\"vocab_size\", 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.embed_in.q_scale\", \"shape\": [\"vocab_size\", 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.0.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.1.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.2.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.3.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.4.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.5.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.6.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.7.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.8.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.9.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.10.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.11.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.12.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.13.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.14.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.15.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.16.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.17.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.18.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.19.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.20.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.21.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.22.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.input_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.input_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.post_attention_layernorm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.post_attention_layernorm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.attention.query_key_value.q_weight\", \"shape\": [3072, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.attention.query_key_value.q_scale\", \"shape\": [3072, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.attention.query_key_value.bias\", \"shape\": [3072], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.attention.dense.q_weight\", \"shape\": [1024, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.attention.dense.q_scale\", \"shape\": [1024, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.attention.dense.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight\", \"shape\": [4096, 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale\", \"shape\": [4096, 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.mlp.dense_h_to_4h.bias\", \"shape\": [4096], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight\", \"shape\": [1024, 512], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale\", \"shape\": [1024, 128], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.layers.23.mlp.dense_4h_to_h.bias\", \"shape\": [1024], \"dtype\": \"float32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.final_layer_norm.weight\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"gpt_neox.final_layer_norm.bias\", \"shape\": [1024], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"embed_out.q_weight\", \"shape\": [\"vocab_size\", 128], \"dtype\": \"uint32\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"embed_out.q_scale\", \"shape\": [\"vocab_size\", 32], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}], \"kv_cache\": {\"num_hidden_layers\": 24, \"num_attention_heads\": 16, \"num_key_value_heads\": 16, \"head_dim\": 64}, \"memory_usage\": {\"alloc_embedding_tensor\": 4194304, \"batch_decode\": 3145728, \"batch_prefill\": 50593792, \"batch_verify\": 50331648, \"create_tir_paged_kv_cache\": 0, \"decode\": 24576, \"embed\": 4194304, \"prefill\": 50333696, \"softmax_with_temperature\": 0}}") @R.function def alloc_embedding_tensor() -> R.Tensor((2048, 1024), dtype="float16"): R.func_attr({"relax.force_pure": True}) shape_heap: R.Object = R.null_value() storage: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv: R.Tensor((2048, 1024), dtype="float16") = R.vm.alloc_tensor(storage, R.prim_value(0), R.shape([2048, 1024]), R.dtype("float16")) R.vm.kill_object(storage) return gv @R.function def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 1024), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[0], param=input_embeds, annotation=R.Tensor((batch_size, 1, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(390), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv272: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv272, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv273: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv273, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv274: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] R.call_packed("vm.builtin.check_tensor_info", gv274, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv275: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] R.call_packed("vm.builtin.check_tensor_info", gv275, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), R.str("ErrorContext(fn=batch_decode, loc=param[0], param=input_embeds, annotation=R.Tensor((batch_size, 1, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv272, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv273, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv274, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv275, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func1(shape_heap) gpt_neox_layers_0_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[3] storage6: R.Object = R.vm.alloc_storage(R.shape([262144]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv276: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc171: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv276, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight4, gpt_neox_layers_0_input_layernorm_bias4, alloc171) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias4) gpt_neox_layers_0_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[8] storage7: R.Object = R.vm.alloc_storage(R.shape([786432]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv277: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc172: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv277, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_0_attention_query_key_value_q_weight4, gpt_neox_layers_0_attention_query_key_value_q_scale4, alloc171, gpt_neox_layers_0_attention_query_key_value_bias4, alloc172) R.vm.kill_object(alloc171) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias4) gv278: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape288: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc172, gv278, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc172) gv279: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape289: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape288, gv279, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape288) gv280: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc173: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv280, R.dtype("float16")) _173: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape289, alloc173) R.vm.kill_object(reshape289) gv281: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape290: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc173, gv281, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc173) gv282: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape291: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape290, gv282, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape290) gpt_neox_layers_0_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[5] gv283: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc174: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv283, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight4, gpt_neox_layers_0_post_attention_layernorm_bias4, alloc174) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias4) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[14] storage8: R.Object = R.vm.alloc_storage(R.shape([1048576]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv284: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc175: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv284, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, alloc174, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, alloc175) R.vm.kill_object(alloc174) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias4) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[17] gv285: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc176: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv285, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, alloc175, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, alloc176) R.vm.kill_object(alloc175) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias4) gpt_neox_layers_0_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[11] gv286: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc177: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv286, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_0_attention_dense_q_weight4, gpt_neox_layers_0_attention_dense_q_scale4, reshape291, gpt_neox_layers_0_attention_dense_bias4, alloc176, input_embeds, alloc177) R.vm.kill_object(reshape291) R.vm.kill_object(alloc176) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias4) gpt_neox_layers_1_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[19] gv287: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc178: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv287, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc177, gpt_neox_layers_1_input_layernorm_weight4, gpt_neox_layers_1_input_layernorm_bias4, alloc178) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias4) gpt_neox_layers_1_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[24] gv288: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc179: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv288, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_1_attention_query_key_value_q_weight4, gpt_neox_layers_1_attention_query_key_value_q_scale4, alloc178, gpt_neox_layers_1_attention_query_key_value_bias4, alloc179) R.vm.kill_object(alloc178) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias4) gv289: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape292: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc179, gv289, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc179) gv290: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape293: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape292, gv290, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape292) gv291: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc180: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv291, R.dtype("float16")) _180: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape293, alloc180) R.vm.kill_object(reshape293) gv292: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape294: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc180, gv292, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc180) gv293: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape295: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape294, gv293, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape294) gpt_neox_layers_1_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[21] gv294: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc181: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv294, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc177, gpt_neox_layers_1_post_attention_layernorm_weight4, gpt_neox_layers_1_post_attention_layernorm_bias4, alloc181) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias4) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[30] storage9: R.Object = R.vm.alloc_storage(R.shape([1048576]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv295: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc182: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv295, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4, alloc181, gpt_neox_layers_1_mlp_dense_h_to_4h_bias4, alloc182) R.vm.kill_object(alloc181) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias4) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[33] gv296: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc183: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv296, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4, alloc182, gpt_neox_layers_1_mlp_dense_4h_to_h_bias4, alloc183) R.vm.kill_object(alloc182) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias4) gpt_neox_layers_1_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[27] gv297: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc184: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv297, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_1_attention_dense_q_weight4, gpt_neox_layers_1_attention_dense_q_scale4, reshape295, gpt_neox_layers_1_attention_dense_bias4, alloc183, alloc177, alloc184) R.vm.kill_object(alloc177) R.vm.kill_object(reshape295) R.vm.kill_object(alloc183) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias4) gpt_neox_layers_2_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[35] gv298: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc185: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv298, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc184, gpt_neox_layers_2_input_layernorm_weight4, gpt_neox_layers_2_input_layernorm_bias4, alloc185) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias4) gpt_neox_layers_2_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[40] gv299: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc186: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv299, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_2_attention_query_key_value_q_weight4, gpt_neox_layers_2_attention_query_key_value_q_scale4, alloc185, gpt_neox_layers_2_attention_query_key_value_bias4, alloc186) R.vm.kill_object(alloc185) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias4) gv300: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape296: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc186, gv300, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc186) gv301: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape297: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape296, gv301, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape296) gv302: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc187: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv302, R.dtype("float16")) _187: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape297, alloc187) R.vm.kill_object(reshape297) gv303: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape298: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc187, gv303, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc187) gv304: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape299: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape298, gv304, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape298) gpt_neox_layers_2_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[37] gv305: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc188: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv305, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc184, gpt_neox_layers_2_post_attention_layernorm_weight4, gpt_neox_layers_2_post_attention_layernorm_bias4, alloc188) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias4) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[46] gv306: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc189: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv306, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4, alloc188, gpt_neox_layers_2_mlp_dense_h_to_4h_bias4, alloc189) R.vm.kill_object(alloc188) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias4) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[49] gv307: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc190: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv307, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4, alloc189, gpt_neox_layers_2_mlp_dense_4h_to_h_bias4, alloc190) R.vm.kill_object(alloc189) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias4) gpt_neox_layers_2_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[43] gv308: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc191: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv308, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_2_attention_dense_q_weight4, gpt_neox_layers_2_attention_dense_q_scale4, reshape299, gpt_neox_layers_2_attention_dense_bias4, alloc190, alloc184, alloc191) R.vm.kill_object(alloc184) R.vm.kill_object(reshape299) R.vm.kill_object(alloc190) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias4) gpt_neox_layers_3_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[51] gv309: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc192: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv309, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc191, gpt_neox_layers_3_input_layernorm_weight4, gpt_neox_layers_3_input_layernorm_bias4, alloc192) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias4) gpt_neox_layers_3_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[56] gv310: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc193: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv310, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_3_attention_query_key_value_q_weight4, gpt_neox_layers_3_attention_query_key_value_q_scale4, alloc192, gpt_neox_layers_3_attention_query_key_value_bias4, alloc193) R.vm.kill_object(alloc192) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias4) gv311: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape300: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc193, gv311, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc193) gv312: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape301: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape300, gv312, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape300) gv313: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc194: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv313, R.dtype("float16")) _194: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape301, alloc194) R.vm.kill_object(reshape301) gv314: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape302: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc194, gv314, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc194) gv315: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape303: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape302, gv315, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape302) gpt_neox_layers_3_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[53] gv316: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc195: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv316, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc191, gpt_neox_layers_3_post_attention_layernorm_weight4, gpt_neox_layers_3_post_attention_layernorm_bias4, alloc195) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias4) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[62] gv317: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc196: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv317, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4, alloc195, gpt_neox_layers_3_mlp_dense_h_to_4h_bias4, alloc196) R.vm.kill_object(alloc195) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias4) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[65] gv318: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc197: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv318, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4, alloc196, gpt_neox_layers_3_mlp_dense_4h_to_h_bias4, alloc197) R.vm.kill_object(alloc196) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias4) gpt_neox_layers_3_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[59] gv319: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc198: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv319, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_3_attention_dense_q_weight4, gpt_neox_layers_3_attention_dense_q_scale4, reshape303, gpt_neox_layers_3_attention_dense_bias4, alloc197, alloc191, alloc198) R.vm.kill_object(alloc191) R.vm.kill_object(reshape303) R.vm.kill_object(alloc197) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias4) gpt_neox_layers_4_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[67] gv320: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc199: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv320, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc198, gpt_neox_layers_4_input_layernorm_weight4, gpt_neox_layers_4_input_layernorm_bias4, alloc199) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias4) gpt_neox_layers_4_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[72] gv321: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc200: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv321, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_4_attention_query_key_value_q_weight4, gpt_neox_layers_4_attention_query_key_value_q_scale4, alloc199, gpt_neox_layers_4_attention_query_key_value_bias4, alloc200) R.vm.kill_object(alloc199) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias4) gv322: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape304: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc200, gv322, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc200) gv323: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape305: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape304, gv323, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape304) gv324: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc201: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv324, R.dtype("float16")) _201: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape305, alloc201) R.vm.kill_object(reshape305) gv325: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape306: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc201, gv325, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc201) gv326: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape307: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape306, gv326, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape306) gpt_neox_layers_4_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[69] gv327: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc202: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv327, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc198, gpt_neox_layers_4_post_attention_layernorm_weight4, gpt_neox_layers_4_post_attention_layernorm_bias4, alloc202) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias4) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[78] gv328: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc203: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv328, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4, alloc202, gpt_neox_layers_4_mlp_dense_h_to_4h_bias4, alloc203) R.vm.kill_object(alloc202) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias4) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[81] gv329: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc204: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv329, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4, alloc203, gpt_neox_layers_4_mlp_dense_4h_to_h_bias4, alloc204) R.vm.kill_object(alloc203) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias4) gpt_neox_layers_4_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[75] gv330: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc205: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv330, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_4_attention_dense_q_weight4, gpt_neox_layers_4_attention_dense_q_scale4, reshape307, gpt_neox_layers_4_attention_dense_bias4, alloc204, alloc198, alloc205) R.vm.kill_object(alloc198) R.vm.kill_object(reshape307) R.vm.kill_object(alloc204) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias4) gpt_neox_layers_5_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[83] gv331: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc206: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv331, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc205, gpt_neox_layers_5_input_layernorm_weight4, gpt_neox_layers_5_input_layernorm_bias4, alloc206) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias4) gpt_neox_layers_5_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[88] gv332: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc207: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv332, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_5_attention_query_key_value_q_weight4, gpt_neox_layers_5_attention_query_key_value_q_scale4, alloc206, gpt_neox_layers_5_attention_query_key_value_bias4, alloc207) R.vm.kill_object(alloc206) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias4) gv333: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape308: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc207, gv333, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc207) gv334: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape309: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape308, gv334, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape308) gv335: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc208: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv335, R.dtype("float16")) _208: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape309, alloc208) R.vm.kill_object(reshape309) gv336: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape310: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc208, gv336, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc208) gv337: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape311: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape310, gv337, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape310) gpt_neox_layers_5_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[85] gv338: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc209: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv338, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc205, gpt_neox_layers_5_post_attention_layernorm_weight4, gpt_neox_layers_5_post_attention_layernorm_bias4, alloc209) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias4) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[94] gv339: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc210: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv339, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4, alloc209, gpt_neox_layers_5_mlp_dense_h_to_4h_bias4, alloc210) R.vm.kill_object(alloc209) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias4) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[97] gv340: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc211: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv340, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4, alloc210, gpt_neox_layers_5_mlp_dense_4h_to_h_bias4, alloc211) R.vm.kill_object(alloc210) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias4) gpt_neox_layers_5_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[91] gv341: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc212: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv341, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_5_attention_dense_q_weight4, gpt_neox_layers_5_attention_dense_q_scale4, reshape311, gpt_neox_layers_5_attention_dense_bias4, alloc211, alloc205, alloc212) R.vm.kill_object(alloc205) R.vm.kill_object(reshape311) R.vm.kill_object(alloc211) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias4) gpt_neox_layers_6_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[98] gpt_neox_layers_6_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[99] gv342: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc213: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv342, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc212, gpt_neox_layers_6_input_layernorm_weight4, gpt_neox_layers_6_input_layernorm_bias4, alloc213) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_bias4) gpt_neox_layers_6_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[102] gpt_neox_layers_6_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[103] gpt_neox_layers_6_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[104] gv343: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc214: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv343, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_6_attention_query_key_value_q_weight4, gpt_neox_layers_6_attention_query_key_value_q_scale4, alloc213, gpt_neox_layers_6_attention_query_key_value_bias4, alloc214) R.vm.kill_object(alloc213) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_bias4) gv344: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape312: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc214, gv344, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc214) gv345: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape313: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape312, gv345, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape312) gv346: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc215: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv346, R.dtype("float16")) _215: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape313, alloc215) R.vm.kill_object(reshape313) gv347: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape314: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc215, gv347, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc215) gv348: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape315: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape314, gv348, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape314) gpt_neox_layers_6_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[100] gpt_neox_layers_6_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[101] gv349: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc216: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv349, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc212, gpt_neox_layers_6_post_attention_layernorm_weight4, gpt_neox_layers_6_post_attention_layernorm_bias4, alloc216) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_bias4) gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[108] gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[109] gpt_neox_layers_6_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[110] gv350: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc217: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv350, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale4, alloc216, gpt_neox_layers_6_mlp_dense_h_to_4h_bias4, alloc217) R.vm.kill_object(alloc216) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_bias4) gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[111] gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[112] gpt_neox_layers_6_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[113] gv351: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc218: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv351, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale4, alloc217, gpt_neox_layers_6_mlp_dense_4h_to_h_bias4, alloc218) R.vm.kill_object(alloc217) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_bias4) gpt_neox_layers_6_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[105] gpt_neox_layers_6_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[106] gpt_neox_layers_6_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[107] gv352: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc219: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv352, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_6_attention_dense_q_weight4, gpt_neox_layers_6_attention_dense_q_scale4, reshape315, gpt_neox_layers_6_attention_dense_bias4, alloc218, alloc212, alloc219) R.vm.kill_object(alloc212) R.vm.kill_object(reshape315) R.vm.kill_object(alloc218) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_6_attention_dense_bias4) gpt_neox_layers_7_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[114] gpt_neox_layers_7_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[115] gv353: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc220: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv353, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc219, gpt_neox_layers_7_input_layernorm_weight4, gpt_neox_layers_7_input_layernorm_bias4, alloc220) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_bias4) gpt_neox_layers_7_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[118] gpt_neox_layers_7_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[119] gpt_neox_layers_7_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[120] gv354: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc221: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv354, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_7_attention_query_key_value_q_weight4, gpt_neox_layers_7_attention_query_key_value_q_scale4, alloc220, gpt_neox_layers_7_attention_query_key_value_bias4, alloc221) R.vm.kill_object(alloc220) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_bias4) gv355: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape316: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc221, gv355, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc221) gv356: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape317: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape316, gv356, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape316) gv357: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc222: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv357, R.dtype("float16")) _222: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape317, alloc222) R.vm.kill_object(reshape317) gv358: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape318: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc222, gv358, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc222) gv359: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape319: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape318, gv359, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape318) gpt_neox_layers_7_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[116] gpt_neox_layers_7_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[117] gv360: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc223: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv360, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc219, gpt_neox_layers_7_post_attention_layernorm_weight4, gpt_neox_layers_7_post_attention_layernorm_bias4, alloc223) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_bias4) gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[124] gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[125] gpt_neox_layers_7_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[126] gv361: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc224: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv361, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale4, alloc223, gpt_neox_layers_7_mlp_dense_h_to_4h_bias4, alloc224) R.vm.kill_object(alloc223) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_bias4) gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[127] gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[128] gpt_neox_layers_7_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[129] gv362: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc225: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv362, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale4, alloc224, gpt_neox_layers_7_mlp_dense_4h_to_h_bias4, alloc225) R.vm.kill_object(alloc224) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_bias4) gpt_neox_layers_7_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[121] gpt_neox_layers_7_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[122] gpt_neox_layers_7_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[123] gv363: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc226: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv363, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_7_attention_dense_q_weight4, gpt_neox_layers_7_attention_dense_q_scale4, reshape319, gpt_neox_layers_7_attention_dense_bias4, alloc225, alloc219, alloc226) R.vm.kill_object(alloc219) R.vm.kill_object(reshape319) R.vm.kill_object(alloc225) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_7_attention_dense_bias4) gpt_neox_layers_8_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[130] gpt_neox_layers_8_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[131] gv364: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc227: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv364, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc226, gpt_neox_layers_8_input_layernorm_weight4, gpt_neox_layers_8_input_layernorm_bias4, alloc227) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_bias4) gpt_neox_layers_8_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[134] gpt_neox_layers_8_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[135] gpt_neox_layers_8_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[136] gv365: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc228: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv365, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_8_attention_query_key_value_q_weight4, gpt_neox_layers_8_attention_query_key_value_q_scale4, alloc227, gpt_neox_layers_8_attention_query_key_value_bias4, alloc228) R.vm.kill_object(alloc227) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_bias4) gv366: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape320: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc228, gv366, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc228) gv367: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape321: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape320, gv367, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape320) gv368: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc229: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv368, R.dtype("float16")) _229: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape321, alloc229) R.vm.kill_object(reshape321) gv369: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape322: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc229, gv369, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc229) gv370: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape323: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape322, gv370, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape322) gpt_neox_layers_8_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[132] gpt_neox_layers_8_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[133] gv371: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc230: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv371, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc226, gpt_neox_layers_8_post_attention_layernorm_weight4, gpt_neox_layers_8_post_attention_layernorm_bias4, alloc230) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_bias4) gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[140] gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[141] gpt_neox_layers_8_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[142] gv372: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc231: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv372, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale4, alloc230, gpt_neox_layers_8_mlp_dense_h_to_4h_bias4, alloc231) R.vm.kill_object(alloc230) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_bias4) gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[143] gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[144] gpt_neox_layers_8_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[145] gv373: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc232: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv373, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale4, alloc231, gpt_neox_layers_8_mlp_dense_4h_to_h_bias4, alloc232) R.vm.kill_object(alloc231) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_bias4) gpt_neox_layers_8_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[137] gpt_neox_layers_8_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[138] gpt_neox_layers_8_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[139] gv374: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc233: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv374, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_8_attention_dense_q_weight4, gpt_neox_layers_8_attention_dense_q_scale4, reshape323, gpt_neox_layers_8_attention_dense_bias4, alloc232, alloc226, alloc233) R.vm.kill_object(alloc226) R.vm.kill_object(reshape323) R.vm.kill_object(alloc232) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_8_attention_dense_bias4) gpt_neox_layers_9_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[146] gpt_neox_layers_9_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[147] gv375: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc234: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv375, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc233, gpt_neox_layers_9_input_layernorm_weight4, gpt_neox_layers_9_input_layernorm_bias4, alloc234) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_bias4) gpt_neox_layers_9_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[150] gpt_neox_layers_9_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[151] gpt_neox_layers_9_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[152] gv376: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc235: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv376, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_9_attention_query_key_value_q_weight4, gpt_neox_layers_9_attention_query_key_value_q_scale4, alloc234, gpt_neox_layers_9_attention_query_key_value_bias4, alloc235) R.vm.kill_object(alloc234) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_bias4) gv377: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape324: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc235, gv377, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc235) gv378: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape325: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape324, gv378, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape324) gv379: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc236: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv379, R.dtype("float16")) _236: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape325, alloc236) R.vm.kill_object(reshape325) gv380: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape326: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc236, gv380, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc236) gv381: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape327: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape326, gv381, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape326) gpt_neox_layers_9_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[148] gpt_neox_layers_9_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[149] gv382: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc237: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv382, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc233, gpt_neox_layers_9_post_attention_layernorm_weight4, gpt_neox_layers_9_post_attention_layernorm_bias4, alloc237) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_bias4) gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[156] gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[157] gpt_neox_layers_9_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[158] gv383: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc238: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv383, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale4, alloc237, gpt_neox_layers_9_mlp_dense_h_to_4h_bias4, alloc238) R.vm.kill_object(alloc237) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_bias4) gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[159] gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[160] gpt_neox_layers_9_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[161] gv384: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc239: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv384, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale4, alloc238, gpt_neox_layers_9_mlp_dense_4h_to_h_bias4, alloc239) R.vm.kill_object(alloc238) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_bias4) gpt_neox_layers_9_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[153] gpt_neox_layers_9_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[154] gpt_neox_layers_9_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[155] gv385: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc240: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv385, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_9_attention_dense_q_weight4, gpt_neox_layers_9_attention_dense_q_scale4, reshape327, gpt_neox_layers_9_attention_dense_bias4, alloc239, alloc233, alloc240) R.vm.kill_object(alloc233) R.vm.kill_object(reshape327) R.vm.kill_object(alloc239) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_9_attention_dense_bias4) gpt_neox_layers_10_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[162] gpt_neox_layers_10_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[163] gv386: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc241: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv386, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc240, gpt_neox_layers_10_input_layernorm_weight4, gpt_neox_layers_10_input_layernorm_bias4, alloc241) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_bias4) gpt_neox_layers_10_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[166] gpt_neox_layers_10_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[167] gpt_neox_layers_10_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[168] gv387: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc242: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv387, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_10_attention_query_key_value_q_weight4, gpt_neox_layers_10_attention_query_key_value_q_scale4, alloc241, gpt_neox_layers_10_attention_query_key_value_bias4, alloc242) R.vm.kill_object(alloc241) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_bias4) gv388: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape328: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc242, gv388, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc242) gv389: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape329: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape328, gv389, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape328) gv390: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc243: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv390, R.dtype("float16")) _243: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape329, alloc243) R.vm.kill_object(reshape329) gv391: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape330: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc243, gv391, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc243) gv392: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape331: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape330, gv392, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape330) gpt_neox_layers_10_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[164] gpt_neox_layers_10_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[165] gv393: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc244: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv393, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc240, gpt_neox_layers_10_post_attention_layernorm_weight4, gpt_neox_layers_10_post_attention_layernorm_bias4, alloc244) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_bias4) gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[172] gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[173] gpt_neox_layers_10_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[174] gv394: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc245: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv394, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale4, alloc244, gpt_neox_layers_10_mlp_dense_h_to_4h_bias4, alloc245) R.vm.kill_object(alloc244) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_bias4) gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[175] gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[176] gpt_neox_layers_10_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[177] gv395: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc246: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv395, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale4, alloc245, gpt_neox_layers_10_mlp_dense_4h_to_h_bias4, alloc246) R.vm.kill_object(alloc245) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_bias4) gpt_neox_layers_10_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[169] gpt_neox_layers_10_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[170] gpt_neox_layers_10_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[171] gv396: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc247: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv396, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_10_attention_dense_q_weight4, gpt_neox_layers_10_attention_dense_q_scale4, reshape331, gpt_neox_layers_10_attention_dense_bias4, alloc246, alloc240, alloc247) R.vm.kill_object(alloc240) R.vm.kill_object(reshape331) R.vm.kill_object(alloc246) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_10_attention_dense_bias4) gpt_neox_layers_11_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[178] gpt_neox_layers_11_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[179] gv397: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc248: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv397, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc247, gpt_neox_layers_11_input_layernorm_weight4, gpt_neox_layers_11_input_layernorm_bias4, alloc248) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_bias4) gpt_neox_layers_11_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[182] gpt_neox_layers_11_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[183] gpt_neox_layers_11_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[184] gv398: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc249: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv398, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_11_attention_query_key_value_q_weight4, gpt_neox_layers_11_attention_query_key_value_q_scale4, alloc248, gpt_neox_layers_11_attention_query_key_value_bias4, alloc249) R.vm.kill_object(alloc248) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_bias4) gv399: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape332: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc249, gv399, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc249) gv400: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape333: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape332, gv400, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape332) gv401: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc250: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv401, R.dtype("float16")) _250: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape333, alloc250) R.vm.kill_object(reshape333) gv402: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape334: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc250, gv402, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc250) gv403: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape335: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape334, gv403, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape334) gpt_neox_layers_11_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[180] gpt_neox_layers_11_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[181] gv404: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc251: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv404, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc247, gpt_neox_layers_11_post_attention_layernorm_weight4, gpt_neox_layers_11_post_attention_layernorm_bias4, alloc251) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_bias4) gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[188] gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[189] gpt_neox_layers_11_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[190] gv405: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc252: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv405, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale4, alloc251, gpt_neox_layers_11_mlp_dense_h_to_4h_bias4, alloc252) R.vm.kill_object(alloc251) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_bias4) gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[191] gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[192] gpt_neox_layers_11_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[193] gv406: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc253: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv406, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale4, alloc252, gpt_neox_layers_11_mlp_dense_4h_to_h_bias4, alloc253) R.vm.kill_object(alloc252) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_bias4) gpt_neox_layers_11_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[185] gpt_neox_layers_11_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[186] gpt_neox_layers_11_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[187] gv407: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc254: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv407, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_11_attention_dense_q_weight4, gpt_neox_layers_11_attention_dense_q_scale4, reshape335, gpt_neox_layers_11_attention_dense_bias4, alloc253, alloc247, alloc254) R.vm.kill_object(alloc247) R.vm.kill_object(reshape335) R.vm.kill_object(alloc253) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_11_attention_dense_bias4) gpt_neox_layers_12_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[194] gpt_neox_layers_12_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[195] gv408: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc255: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv408, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc254, gpt_neox_layers_12_input_layernorm_weight4, gpt_neox_layers_12_input_layernorm_bias4, alloc255) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_bias4) gpt_neox_layers_12_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[198] gpt_neox_layers_12_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[199] gpt_neox_layers_12_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[200] gv409: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc256: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv409, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_12_attention_query_key_value_q_weight4, gpt_neox_layers_12_attention_query_key_value_q_scale4, alloc255, gpt_neox_layers_12_attention_query_key_value_bias4, alloc256) R.vm.kill_object(alloc255) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_bias4) gv410: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape336: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc256, gv410, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc256) gv411: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape337: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape336, gv411, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape336) gv412: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc257: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv412, R.dtype("float16")) _257: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape337, alloc257) R.vm.kill_object(reshape337) gv413: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape338: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc257, gv413, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc257) gv414: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape339: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape338, gv414, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape338) gpt_neox_layers_12_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[196] gpt_neox_layers_12_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[197] gv415: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc258: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv415, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc254, gpt_neox_layers_12_post_attention_layernorm_weight4, gpt_neox_layers_12_post_attention_layernorm_bias4, alloc258) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_bias4) gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[204] gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[205] gpt_neox_layers_12_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[206] gv416: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc259: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv416, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale4, alloc258, gpt_neox_layers_12_mlp_dense_h_to_4h_bias4, alloc259) R.vm.kill_object(alloc258) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_bias4) gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[207] gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[208] gpt_neox_layers_12_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[209] gv417: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc260: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv417, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale4, alloc259, gpt_neox_layers_12_mlp_dense_4h_to_h_bias4, alloc260) R.vm.kill_object(alloc259) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_bias4) gpt_neox_layers_12_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[201] gpt_neox_layers_12_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[202] gpt_neox_layers_12_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[203] gv418: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc261: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv418, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_12_attention_dense_q_weight4, gpt_neox_layers_12_attention_dense_q_scale4, reshape339, gpt_neox_layers_12_attention_dense_bias4, alloc260, alloc254, alloc261) R.vm.kill_object(alloc254) R.vm.kill_object(reshape339) R.vm.kill_object(alloc260) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_12_attention_dense_bias4) gpt_neox_layers_13_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[210] gpt_neox_layers_13_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[211] gv419: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc262: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv419, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc261, gpt_neox_layers_13_input_layernorm_weight4, gpt_neox_layers_13_input_layernorm_bias4, alloc262) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_bias4) gpt_neox_layers_13_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[214] gpt_neox_layers_13_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[215] gpt_neox_layers_13_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[216] gv420: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc263: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv420, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_13_attention_query_key_value_q_weight4, gpt_neox_layers_13_attention_query_key_value_q_scale4, alloc262, gpt_neox_layers_13_attention_query_key_value_bias4, alloc263) R.vm.kill_object(alloc262) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_bias4) gv421: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape340: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc263, gv421, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc263) gv422: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape341: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape340, gv422, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape340) gv423: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc264: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv423, R.dtype("float16")) _264: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape341, alloc264) R.vm.kill_object(reshape341) gv424: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape342: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc264, gv424, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc264) gv425: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape343: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape342, gv425, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape342) gpt_neox_layers_13_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[212] gpt_neox_layers_13_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[213] gv426: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc265: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv426, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc261, gpt_neox_layers_13_post_attention_layernorm_weight4, gpt_neox_layers_13_post_attention_layernorm_bias4, alloc265) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_bias4) gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[220] gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[221] gpt_neox_layers_13_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[222] gv427: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc266: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv427, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale4, alloc265, gpt_neox_layers_13_mlp_dense_h_to_4h_bias4, alloc266) R.vm.kill_object(alloc265) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_bias4) gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[223] gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[224] gpt_neox_layers_13_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[225] gv428: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc267: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv428, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale4, alloc266, gpt_neox_layers_13_mlp_dense_4h_to_h_bias4, alloc267) R.vm.kill_object(alloc266) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_bias4) gpt_neox_layers_13_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[217] gpt_neox_layers_13_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[218] gpt_neox_layers_13_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[219] gv429: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc268: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv429, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_13_attention_dense_q_weight4, gpt_neox_layers_13_attention_dense_q_scale4, reshape343, gpt_neox_layers_13_attention_dense_bias4, alloc267, alloc261, alloc268) R.vm.kill_object(alloc261) R.vm.kill_object(reshape343) R.vm.kill_object(alloc267) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_13_attention_dense_bias4) gpt_neox_layers_14_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[226] gpt_neox_layers_14_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[227] gv430: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc269: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv430, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc268, gpt_neox_layers_14_input_layernorm_weight4, gpt_neox_layers_14_input_layernorm_bias4, alloc269) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_bias4) gpt_neox_layers_14_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[230] gpt_neox_layers_14_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[231] gpt_neox_layers_14_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[232] gv431: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc270: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv431, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_14_attention_query_key_value_q_weight4, gpt_neox_layers_14_attention_query_key_value_q_scale4, alloc269, gpt_neox_layers_14_attention_query_key_value_bias4, alloc270) R.vm.kill_object(alloc269) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_bias4) gv432: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape344: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc270, gv432, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc270) gv433: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape345: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape344, gv433, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape344) gv434: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc271: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv434, R.dtype("float16")) _271: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape345, alloc271) R.vm.kill_object(reshape345) gv435: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape346: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc271, gv435, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc271) gv436: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape347: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape346, gv436, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape346) gpt_neox_layers_14_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[228] gpt_neox_layers_14_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[229] gv437: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc272: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv437, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc268, gpt_neox_layers_14_post_attention_layernorm_weight4, gpt_neox_layers_14_post_attention_layernorm_bias4, alloc272) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_bias4) gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[236] gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[237] gpt_neox_layers_14_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[238] gv438: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc273: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv438, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale4, alloc272, gpt_neox_layers_14_mlp_dense_h_to_4h_bias4, alloc273) R.vm.kill_object(alloc272) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_bias4) gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[239] gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[240] gpt_neox_layers_14_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[241] gv439: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc274: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv439, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale4, alloc273, gpt_neox_layers_14_mlp_dense_4h_to_h_bias4, alloc274) R.vm.kill_object(alloc273) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_bias4) gpt_neox_layers_14_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[233] gpt_neox_layers_14_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[234] gpt_neox_layers_14_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[235] gv440: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc275: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv440, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_14_attention_dense_q_weight4, gpt_neox_layers_14_attention_dense_q_scale4, reshape347, gpt_neox_layers_14_attention_dense_bias4, alloc274, alloc268, alloc275) R.vm.kill_object(alloc268) R.vm.kill_object(reshape347) R.vm.kill_object(alloc274) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_14_attention_dense_bias4) gpt_neox_layers_15_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[242] gpt_neox_layers_15_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[243] gv441: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc276: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv441, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc275, gpt_neox_layers_15_input_layernorm_weight4, gpt_neox_layers_15_input_layernorm_bias4, alloc276) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_bias4) gpt_neox_layers_15_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[246] gpt_neox_layers_15_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[247] gpt_neox_layers_15_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[248] gv442: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc277: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv442, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_15_attention_query_key_value_q_weight4, gpt_neox_layers_15_attention_query_key_value_q_scale4, alloc276, gpt_neox_layers_15_attention_query_key_value_bias4, alloc277) R.vm.kill_object(alloc276) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_bias4) gv443: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape348: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc277, gv443, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc277) gv444: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape349: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape348, gv444, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape348) gv445: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc278: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv445, R.dtype("float16")) _278: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape349, alloc278) R.vm.kill_object(reshape349) gv446: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape350: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc278, gv446, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc278) gv447: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape351: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape350, gv447, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape350) gpt_neox_layers_15_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[244] gpt_neox_layers_15_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[245] gv448: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc279: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv448, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc275, gpt_neox_layers_15_post_attention_layernorm_weight4, gpt_neox_layers_15_post_attention_layernorm_bias4, alloc279) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_bias4) gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[252] gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[253] gpt_neox_layers_15_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[254] gv449: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc280: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv449, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale4, alloc279, gpt_neox_layers_15_mlp_dense_h_to_4h_bias4, alloc280) R.vm.kill_object(alloc279) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_bias4) gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[255] gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[256] gpt_neox_layers_15_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[257] gv450: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc281: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv450, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale4, alloc280, gpt_neox_layers_15_mlp_dense_4h_to_h_bias4, alloc281) R.vm.kill_object(alloc280) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_bias4) gpt_neox_layers_15_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[249] gpt_neox_layers_15_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[250] gpt_neox_layers_15_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[251] gv451: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc282: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv451, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_15_attention_dense_q_weight4, gpt_neox_layers_15_attention_dense_q_scale4, reshape351, gpt_neox_layers_15_attention_dense_bias4, alloc281, alloc275, alloc282) R.vm.kill_object(alloc275) R.vm.kill_object(reshape351) R.vm.kill_object(alloc281) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_15_attention_dense_bias4) gpt_neox_layers_16_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[258] gpt_neox_layers_16_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[259] gv452: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc283: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv452, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc282, gpt_neox_layers_16_input_layernorm_weight4, gpt_neox_layers_16_input_layernorm_bias4, alloc283) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_bias4) gpt_neox_layers_16_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[262] gpt_neox_layers_16_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[263] gpt_neox_layers_16_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[264] gv453: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc284: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv453, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_16_attention_query_key_value_q_weight4, gpt_neox_layers_16_attention_query_key_value_q_scale4, alloc283, gpt_neox_layers_16_attention_query_key_value_bias4, alloc284) R.vm.kill_object(alloc283) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_bias4) gv454: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape352: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc284, gv454, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc284) gv455: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape353: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape352, gv455, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape352) gv456: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc285: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv456, R.dtype("float16")) _285: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape353, alloc285) R.vm.kill_object(reshape353) gv457: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape354: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc285, gv457, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc285) gv458: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape355: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape354, gv458, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape354) gpt_neox_layers_16_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[260] gpt_neox_layers_16_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[261] gv459: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc286: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv459, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc282, gpt_neox_layers_16_post_attention_layernorm_weight4, gpt_neox_layers_16_post_attention_layernorm_bias4, alloc286) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_bias4) gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[268] gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[269] gpt_neox_layers_16_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[270] gv460: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc287: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv460, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale4, alloc286, gpt_neox_layers_16_mlp_dense_h_to_4h_bias4, alloc287) R.vm.kill_object(alloc286) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_bias4) gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[271] gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[272] gpt_neox_layers_16_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[273] gv461: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc288: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv461, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale4, alloc287, gpt_neox_layers_16_mlp_dense_4h_to_h_bias4, alloc288) R.vm.kill_object(alloc287) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_bias4) gpt_neox_layers_16_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[265] gpt_neox_layers_16_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[266] gpt_neox_layers_16_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[267] gv462: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc289: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv462, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_16_attention_dense_q_weight4, gpt_neox_layers_16_attention_dense_q_scale4, reshape355, gpt_neox_layers_16_attention_dense_bias4, alloc288, alloc282, alloc289) R.vm.kill_object(alloc282) R.vm.kill_object(reshape355) R.vm.kill_object(alloc288) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_16_attention_dense_bias4) gpt_neox_layers_17_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[274] gpt_neox_layers_17_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[275] gv463: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc290: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv463, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc289, gpt_neox_layers_17_input_layernorm_weight4, gpt_neox_layers_17_input_layernorm_bias4, alloc290) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_bias4) gpt_neox_layers_17_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[278] gpt_neox_layers_17_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[279] gpt_neox_layers_17_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[280] gv464: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc291: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv464, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_17_attention_query_key_value_q_weight4, gpt_neox_layers_17_attention_query_key_value_q_scale4, alloc290, gpt_neox_layers_17_attention_query_key_value_bias4, alloc291) R.vm.kill_object(alloc290) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_bias4) gv465: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape356: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc291, gv465, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc291) gv466: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape357: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape356, gv466, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape356) gv467: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc292: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv467, R.dtype("float16")) _292: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape357, alloc292) R.vm.kill_object(reshape357) gv468: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape358: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc292, gv468, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc292) gv469: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape359: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape358, gv469, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape358) gpt_neox_layers_17_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[276] gpt_neox_layers_17_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[277] gv470: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc293: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv470, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc289, gpt_neox_layers_17_post_attention_layernorm_weight4, gpt_neox_layers_17_post_attention_layernorm_bias4, alloc293) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_bias4) gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[284] gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[285] gpt_neox_layers_17_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[286] gv471: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc294: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv471, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale4, alloc293, gpt_neox_layers_17_mlp_dense_h_to_4h_bias4, alloc294) R.vm.kill_object(alloc293) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_bias4) gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[287] gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[288] gpt_neox_layers_17_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[289] gv472: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc295: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv472, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale4, alloc294, gpt_neox_layers_17_mlp_dense_4h_to_h_bias4, alloc295) R.vm.kill_object(alloc294) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_bias4) gpt_neox_layers_17_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[281] gpt_neox_layers_17_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[282] gpt_neox_layers_17_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[283] gv473: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc296: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv473, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_17_attention_dense_q_weight4, gpt_neox_layers_17_attention_dense_q_scale4, reshape359, gpt_neox_layers_17_attention_dense_bias4, alloc295, alloc289, alloc296) R.vm.kill_object(alloc289) R.vm.kill_object(reshape359) R.vm.kill_object(alloc295) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_17_attention_dense_bias4) gpt_neox_layers_18_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[290] gpt_neox_layers_18_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[291] gv474: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc297: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv474, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc296, gpt_neox_layers_18_input_layernorm_weight4, gpt_neox_layers_18_input_layernorm_bias4, alloc297) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_bias4) gpt_neox_layers_18_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[294] gpt_neox_layers_18_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[295] gpt_neox_layers_18_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[296] gv475: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc298: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv475, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_18_attention_query_key_value_q_weight4, gpt_neox_layers_18_attention_query_key_value_q_scale4, alloc297, gpt_neox_layers_18_attention_query_key_value_bias4, alloc298) R.vm.kill_object(alloc297) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_bias4) gv476: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape360: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc298, gv476, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc298) gv477: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape361: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape360, gv477, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape360) gv478: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc299: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv478, R.dtype("float16")) _299: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape361, alloc299) R.vm.kill_object(reshape361) gv479: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape362: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc299, gv479, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc299) gv480: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape363: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape362, gv480, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape362) gpt_neox_layers_18_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[292] gpt_neox_layers_18_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[293] gv481: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc300: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv481, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc296, gpt_neox_layers_18_post_attention_layernorm_weight4, gpt_neox_layers_18_post_attention_layernorm_bias4, alloc300) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_bias4) gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[300] gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[301] gpt_neox_layers_18_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[302] gv482: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc301: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv482, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale4, alloc300, gpt_neox_layers_18_mlp_dense_h_to_4h_bias4, alloc301) R.vm.kill_object(alloc300) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_bias4) gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[303] gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[304] gpt_neox_layers_18_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[305] gv483: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc302: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv483, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale4, alloc301, gpt_neox_layers_18_mlp_dense_4h_to_h_bias4, alloc302) R.vm.kill_object(alloc301) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_bias4) gpt_neox_layers_18_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[297] gpt_neox_layers_18_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[298] gpt_neox_layers_18_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[299] gv484: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc303: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv484, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_18_attention_dense_q_weight4, gpt_neox_layers_18_attention_dense_q_scale4, reshape363, gpt_neox_layers_18_attention_dense_bias4, alloc302, alloc296, alloc303) R.vm.kill_object(alloc296) R.vm.kill_object(reshape363) R.vm.kill_object(alloc302) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_18_attention_dense_bias4) gpt_neox_layers_19_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[306] gpt_neox_layers_19_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[307] gv485: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc304: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv485, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc303, gpt_neox_layers_19_input_layernorm_weight4, gpt_neox_layers_19_input_layernorm_bias4, alloc304) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_bias4) gpt_neox_layers_19_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[310] gpt_neox_layers_19_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[311] gpt_neox_layers_19_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[312] gv486: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc305: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv486, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_19_attention_query_key_value_q_weight4, gpt_neox_layers_19_attention_query_key_value_q_scale4, alloc304, gpt_neox_layers_19_attention_query_key_value_bias4, alloc305) R.vm.kill_object(alloc304) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_bias4) gv487: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape364: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc305, gv487, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc305) gv488: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape365: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape364, gv488, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape364) gv489: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc306: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv489, R.dtype("float16")) _306: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape365, alloc306) R.vm.kill_object(reshape365) gv490: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape366: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc306, gv490, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc306) gv491: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape367: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape366, gv491, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape366) gpt_neox_layers_19_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[308] gpt_neox_layers_19_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[309] gv492: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc307: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv492, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc303, gpt_neox_layers_19_post_attention_layernorm_weight4, gpt_neox_layers_19_post_attention_layernorm_bias4, alloc307) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_bias4) gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[316] gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[317] gpt_neox_layers_19_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[318] gv493: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc308: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv493, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale4, alloc307, gpt_neox_layers_19_mlp_dense_h_to_4h_bias4, alloc308) R.vm.kill_object(alloc307) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_bias4) gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[319] gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[320] gpt_neox_layers_19_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[321] gv494: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc309: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv494, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale4, alloc308, gpt_neox_layers_19_mlp_dense_4h_to_h_bias4, alloc309) R.vm.kill_object(alloc308) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_bias4) gpt_neox_layers_19_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[313] gpt_neox_layers_19_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[314] gpt_neox_layers_19_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[315] gv495: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc310: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv495, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_19_attention_dense_q_weight4, gpt_neox_layers_19_attention_dense_q_scale4, reshape367, gpt_neox_layers_19_attention_dense_bias4, alloc309, alloc303, alloc310) R.vm.kill_object(alloc303) R.vm.kill_object(reshape367) R.vm.kill_object(alloc309) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_19_attention_dense_bias4) gpt_neox_layers_20_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[322] gpt_neox_layers_20_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[323] gv496: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc311: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv496, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc310, gpt_neox_layers_20_input_layernorm_weight4, gpt_neox_layers_20_input_layernorm_bias4, alloc311) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_bias4) gpt_neox_layers_20_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[326] gpt_neox_layers_20_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[327] gpt_neox_layers_20_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[328] gv497: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc312: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv497, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_20_attention_query_key_value_q_weight4, gpt_neox_layers_20_attention_query_key_value_q_scale4, alloc311, gpt_neox_layers_20_attention_query_key_value_bias4, alloc312) R.vm.kill_object(alloc311) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_bias4) gv498: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape368: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc312, gv498, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc312) gv499: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape369: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape368, gv499, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape368) gv500: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc313: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv500, R.dtype("float16")) _313: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape369, alloc313) R.vm.kill_object(reshape369) gv501: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape370: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc313, gv501, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc313) gv502: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape371: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape370, gv502, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape370) gpt_neox_layers_20_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[324] gpt_neox_layers_20_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[325] gv503: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc314: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv503, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc310, gpt_neox_layers_20_post_attention_layernorm_weight4, gpt_neox_layers_20_post_attention_layernorm_bias4, alloc314) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_bias4) gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[332] gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[333] gpt_neox_layers_20_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[334] gv504: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc315: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv504, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale4, alloc314, gpt_neox_layers_20_mlp_dense_h_to_4h_bias4, alloc315) R.vm.kill_object(alloc314) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_bias4) gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[335] gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[336] gpt_neox_layers_20_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[337] gv505: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc316: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv505, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale4, alloc315, gpt_neox_layers_20_mlp_dense_4h_to_h_bias4, alloc316) R.vm.kill_object(alloc315) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_bias4) gpt_neox_layers_20_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[329] gpt_neox_layers_20_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[330] gpt_neox_layers_20_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[331] gv506: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc317: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv506, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_20_attention_dense_q_weight4, gpt_neox_layers_20_attention_dense_q_scale4, reshape371, gpt_neox_layers_20_attention_dense_bias4, alloc316, alloc310, alloc317) R.vm.kill_object(alloc310) R.vm.kill_object(reshape371) R.vm.kill_object(alloc316) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_20_attention_dense_bias4) gpt_neox_layers_21_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[338] gpt_neox_layers_21_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[339] gv507: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc318: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv507, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc317, gpt_neox_layers_21_input_layernorm_weight4, gpt_neox_layers_21_input_layernorm_bias4, alloc318) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_bias4) gpt_neox_layers_21_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[342] gpt_neox_layers_21_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[343] gpt_neox_layers_21_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[344] gv508: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc319: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv508, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_21_attention_query_key_value_q_weight4, gpt_neox_layers_21_attention_query_key_value_q_scale4, alloc318, gpt_neox_layers_21_attention_query_key_value_bias4, alloc319) R.vm.kill_object(alloc318) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_bias4) gv509: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape372: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc319, gv509, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc319) gv510: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape373: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape372, gv510, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape372) gv511: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc320: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv511, R.dtype("float16")) _320: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape373, alloc320) R.vm.kill_object(reshape373) gv512: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape374: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc320, gv512, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc320) gv513: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape375: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape374, gv513, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape374) gpt_neox_layers_21_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[340] gpt_neox_layers_21_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[341] gv514: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc321: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv514, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc317, gpt_neox_layers_21_post_attention_layernorm_weight4, gpt_neox_layers_21_post_attention_layernorm_bias4, alloc321) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_bias4) gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[348] gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[349] gpt_neox_layers_21_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[350] gv515: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc322: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv515, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale4, alloc321, gpt_neox_layers_21_mlp_dense_h_to_4h_bias4, alloc322) R.vm.kill_object(alloc321) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_bias4) gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[351] gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[352] gpt_neox_layers_21_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[353] gv516: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc323: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv516, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale4, alloc322, gpt_neox_layers_21_mlp_dense_4h_to_h_bias4, alloc323) R.vm.kill_object(alloc322) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_bias4) gpt_neox_layers_21_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[345] gpt_neox_layers_21_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[346] gpt_neox_layers_21_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[347] gv517: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc324: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv517, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_21_attention_dense_q_weight4, gpt_neox_layers_21_attention_dense_q_scale4, reshape375, gpt_neox_layers_21_attention_dense_bias4, alloc323, alloc317, alloc324) R.vm.kill_object(alloc317) R.vm.kill_object(reshape375) R.vm.kill_object(alloc323) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_21_attention_dense_bias4) gpt_neox_layers_22_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[354] gpt_neox_layers_22_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[355] gv518: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc325: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv518, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc324, gpt_neox_layers_22_input_layernorm_weight4, gpt_neox_layers_22_input_layernorm_bias4, alloc325) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_bias4) gpt_neox_layers_22_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[358] gpt_neox_layers_22_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[359] gpt_neox_layers_22_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[360] gv519: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc326: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv519, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_22_attention_query_key_value_q_weight4, gpt_neox_layers_22_attention_query_key_value_q_scale4, alloc325, gpt_neox_layers_22_attention_query_key_value_bias4, alloc326) R.vm.kill_object(alloc325) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_bias4) gv520: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape376: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc326, gv520, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc326) gv521: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape377: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape376, gv521, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape376) gv522: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc327: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv522, R.dtype("float16")) _327: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape377, alloc327) R.vm.kill_object(reshape377) gv523: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape378: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc327, gv523, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc327) gv524: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape379: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape378, gv524, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape378) gpt_neox_layers_22_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[356] gpt_neox_layers_22_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[357] gv525: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc328: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv525, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc324, gpt_neox_layers_22_post_attention_layernorm_weight4, gpt_neox_layers_22_post_attention_layernorm_bias4, alloc328) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_bias4) gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[364] gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[365] gpt_neox_layers_22_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[366] gv526: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc329: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv526, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale4, alloc328, gpt_neox_layers_22_mlp_dense_h_to_4h_bias4, alloc329) R.vm.kill_object(alloc328) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_bias4) gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[367] gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[368] gpt_neox_layers_22_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[369] gv527: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc330: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv527, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale4, alloc329, gpt_neox_layers_22_mlp_dense_4h_to_h_bias4, alloc330) R.vm.kill_object(alloc329) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_bias4) gpt_neox_layers_22_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[361] gpt_neox_layers_22_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[362] gpt_neox_layers_22_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[363] gv528: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc331: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv528, R.dtype("float16")) R.vm.kill_object(storage8) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_22_attention_dense_q_weight4, gpt_neox_layers_22_attention_dense_q_scale4, reshape379, gpt_neox_layers_22_attention_dense_bias4, alloc330, alloc324, alloc331) R.vm.kill_object(alloc324) R.vm.kill_object(reshape379) R.vm.kill_object(alloc330) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_22_attention_dense_bias4) gpt_neox_layers_23_input_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[370] gpt_neox_layers_23_input_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[371] gv529: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc332: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv529, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc331, gpt_neox_layers_23_input_layernorm_weight4, gpt_neox_layers_23_input_layernorm_bias4, alloc332) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_bias4) gpt_neox_layers_23_attention_query_key_value_q_weight4: R.Tensor((3072, 128), dtype="uint32") = packed_params[374] gpt_neox_layers_23_attention_query_key_value_q_scale4: R.Tensor((3072, 32), dtype="float16") = packed_params[375] gpt_neox_layers_23_attention_query_key_value_bias4: R.Tensor((3072,), dtype="float16") = packed_params[376] gv530: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc333: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv530, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add(gpt_neox_layers_23_attention_query_key_value_q_weight4, gpt_neox_layers_23_attention_query_key_value_q_scale4, alloc332, gpt_neox_layers_23_attention_query_key_value_bias4, alloc333) R.vm.kill_object(alloc332) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_weight4) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_scale4) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_bias4) gv531: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape380: R.Tensor((batch_size, 1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc333, gv531, sinfo_args=(R.Tensor((batch_size, 1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc333) gv532: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape381: R.Tensor((batch_size, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape380, gv532, sinfo_args=(R.Tensor((batch_size, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape380) gv533: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc334: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv533, R.dtype("float16")) _334: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape381, alloc334) R.vm.kill_object(reshape381) gv534: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape382: R.Tensor((batch_size, 1, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc334, gv534, sinfo_args=(R.Tensor((batch_size, 1, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc334) gv535: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape383: R.Tensor((batch_size, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape382, gv535, sinfo_args=(R.Tensor((batch_size, 1, 1024), dtype="float16"),)) R.vm.kill_object(reshape382) gpt_neox_layers_23_post_attention_layernorm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[372] gpt_neox_layers_23_post_attention_layernorm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[373] gv536: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc335: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv536, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm(alloc331, gpt_neox_layers_23_post_attention_layernorm_weight4, gpt_neox_layers_23_post_attention_layernorm_bias4, alloc335) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_weight4) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_bias4) gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight4: R.Tensor((4096, 128), dtype="uint32") = packed_params[380] gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale4: R.Tensor((4096, 32), dtype="float16") = packed_params[381] gpt_neox_layers_23_mlp_dense_h_to_4h_bias4: R.Tensor((4096,), dtype="float32") = packed_params[382] gv537: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc336: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv537, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale4, alloc335, gpt_neox_layers_23_mlp_dense_h_to_4h_bias4, alloc336) R.vm.kill_object(alloc335) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight4) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale4) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_bias4) gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight4: R.Tensor((1024, 512), dtype="uint32") = packed_params[383] gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale4: R.Tensor((1024, 128), dtype="float16") = packed_params[384] gpt_neox_layers_23_mlp_dense_4h_to_h_bias4: R.Tensor((1024,), dtype="float32") = packed_params[385] gv538: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc337: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv538, R.dtype("float16")) R.vm.kill_object(storage7) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale4, alloc336, gpt_neox_layers_23_mlp_dense_4h_to_h_bias4, alloc337) R.vm.kill_object(alloc336) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight4) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale4) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_bias4) gpt_neox_layers_23_attention_dense_q_weight4: R.Tensor((1024, 128), dtype="uint32") = packed_params[377] gpt_neox_layers_23_attention_dense_q_scale4: R.Tensor((1024, 32), dtype="float16") = packed_params[378] gpt_neox_layers_23_attention_dense_bias4: R.Tensor((1024,), dtype="float16") = packed_params[379] gv539: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc338: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv539, R.dtype("float16")) R.vm.kill_object(storage9) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4(gpt_neox_layers_23_attention_dense_q_weight4, gpt_neox_layers_23_attention_dense_q_scale4, reshape383, gpt_neox_layers_23_attention_dense_bias4, alloc337, alloc331, alloc338) R.vm.kill_object(alloc331) R.vm.kill_object(reshape383) R.vm.kill_object(alloc337) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_weight4) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_scale4) R.vm.kill_object(gpt_neox_layers_23_attention_dense_bias4) gpt_neox_final_layer_norm_weight4: R.Tensor((1024,), dtype="float16") = packed_params[386] gpt_neox_final_layer_norm_bias4: R.Tensor((1024,), dtype="float16") = packed_params[387] gv540: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc339: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv540, R.dtype("float16")) R.vm.kill_object(storage6) cls.gpt_neox_q4f16_1_layer_norm(alloc338, gpt_neox_final_layer_norm_weight4, gpt_neox_final_layer_norm_bias4, alloc339) R.vm.kill_object(alloc338) R.vm.kill_object(gpt_neox_final_layer_norm_weight4) R.vm.kill_object(gpt_neox_final_layer_norm_bias4) embed_out_q_weight4: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] embed_out_q_scale4: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] gv541: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage10: R.Object = R.vm.alloc_storage(gv541, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv542: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc340: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv542, R.dtype("float32")) R.vm.kill_object(storage10) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2(embed_out_q_weight4, embed_out_q_scale4, alloc339, alloc340) R.vm.kill_object(alloc339) R.vm.kill_object(embed_out_q_weight4) R.vm.kill_object(embed_out_q_scale4) gv4: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc340, paged_kv_cache R.vm.kill_object(alloc340) gv543: R.Tensor(dtype="float32", ndim=3) = gv4[0] R.call_packed("vm.builtin.match_shape", gv543, shape_heap, R.prim_value(3), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=batch_decode, loc=return, annotation=R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv4 @R.function def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 1024), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tensor_info", logit_positions, R.prim_value(1), R.dtype("int32"), R.str("ErrorContext(fn=batch_prefill, loc=param[1], param=logit_positions, annotation=R.Tensor((batch_size,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(390), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv837: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv837, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv838: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv838, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv839: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] R.call_packed("vm.builtin.check_tensor_info", gv839, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv840: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] R.call_packed("vm.builtin.check_tensor_info", gv840, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), R.str("ErrorContext(fn=batch_prefill, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", logit_positions, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=batch_prefill, loc=param[1], param=logit_positions, annotation=R.Tensor((batch_size,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv837, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(2), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv838, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(2), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv839, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(2), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv840, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(2), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func5(shape_heap) gpt_neox_layers_0_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[3] storage25: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv841: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc685: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv841, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_input_layernorm_weight3, gpt_neox_layers_0_input_layernorm_bias3, alloc685) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias3) gpt_neox_layers_0_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[8] storage26: R.Object = R.vm.alloc_storage(R.shape([12582912]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv842: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc686: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv842, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight3, gpt_neox_layers_0_attention_query_key_value_q_scale3, alloc685, gpt_neox_layers_0_attention_query_key_value_bias3, alloc686) R.vm.kill_object(alloc685) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias3) gv843: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape192: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc686, gv843, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc686) gv844: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape193: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape192, gv844, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape192) gv845: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc687: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv845, R.dtype("float16")) _686: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape193, alloc687) R.vm.kill_object(reshape193) gv846: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape194: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc687, gv846, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc687) gv847: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape195: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape194, gv847, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape194) gpt_neox_layers_0_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[5] gv848: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc688: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv848, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight3, gpt_neox_layers_0_post_attention_layernorm_bias3, alloc688) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias3) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[14] storage27: R.Object = R.vm.alloc_storage(R.shape([16777216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv849: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc689: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv849, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, alloc688, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, alloc689) R.vm.kill_object(alloc688) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias3) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[17] gv850: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc690: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv850, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, alloc689, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, alloc690) R.vm.kill_object(alloc689) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias3) gpt_neox_layers_0_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[11] gv851: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc691: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv851, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight3, gpt_neox_layers_0_attention_dense_q_scale3, reshape195, gpt_neox_layers_0_attention_dense_bias3, alloc690, input_embeds, alloc691) R.vm.kill_object(reshape195) R.vm.kill_object(alloc690) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias3) gpt_neox_layers_1_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[19] gv852: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc692: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv852, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc691, gpt_neox_layers_1_input_layernorm_weight3, gpt_neox_layers_1_input_layernorm_bias3, alloc692) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias3) gpt_neox_layers_1_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[24] gv853: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc693: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv853, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_1_attention_query_key_value_q_weight3, gpt_neox_layers_1_attention_query_key_value_q_scale3, alloc692, gpt_neox_layers_1_attention_query_key_value_bias3, alloc693) R.vm.kill_object(alloc692) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias3) gv854: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape196: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc693, gv854, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc693) gv855: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape197: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape196, gv855, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape196) gv856: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc694: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv856, R.dtype("float16")) _693: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape197, alloc694) R.vm.kill_object(reshape197) gv857: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape198: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc694, gv857, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc694) gv858: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape199: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape198, gv858, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape198) gpt_neox_layers_1_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[21] gv859: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc695: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv859, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc691, gpt_neox_layers_1_post_attention_layernorm_weight3, gpt_neox_layers_1_post_attention_layernorm_bias3, alloc695) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias3) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[30] storage28: R.Object = R.vm.alloc_storage(R.shape([16777216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv860: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc696: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv860, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3, alloc695, gpt_neox_layers_1_mlp_dense_h_to_4h_bias3, alloc696) R.vm.kill_object(alloc695) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias3) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[33] gv861: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc697: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv861, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3, alloc696, gpt_neox_layers_1_mlp_dense_4h_to_h_bias3, alloc697) R.vm.kill_object(alloc696) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias3) gpt_neox_layers_1_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[27] gv862: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc698: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv862, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_1_attention_dense_q_weight3, gpt_neox_layers_1_attention_dense_q_scale3, reshape199, gpt_neox_layers_1_attention_dense_bias3, alloc697, alloc691, alloc698) R.vm.kill_object(alloc691) R.vm.kill_object(reshape199) R.vm.kill_object(alloc697) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias3) gpt_neox_layers_2_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[35] gv863: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc699: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv863, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc698, gpt_neox_layers_2_input_layernorm_weight3, gpt_neox_layers_2_input_layernorm_bias3, alloc699) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias3) gpt_neox_layers_2_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[40] gv864: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc700: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv864, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_2_attention_query_key_value_q_weight3, gpt_neox_layers_2_attention_query_key_value_q_scale3, alloc699, gpt_neox_layers_2_attention_query_key_value_bias3, alloc700) R.vm.kill_object(alloc699) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias3) gv865: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape200: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc700, gv865, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc700) gv866: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape201: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape200, gv866, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape200) gv867: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc701: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv867, R.dtype("float16")) _700: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape201, alloc701) R.vm.kill_object(reshape201) gv868: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape202: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc701, gv868, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc701) gv869: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape203: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape202, gv869, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape202) gpt_neox_layers_2_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[37] gv870: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc702: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv870, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc698, gpt_neox_layers_2_post_attention_layernorm_weight3, gpt_neox_layers_2_post_attention_layernorm_bias3, alloc702) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias3) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[46] gv871: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc703: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv871, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3, alloc702, gpt_neox_layers_2_mlp_dense_h_to_4h_bias3, alloc703) R.vm.kill_object(alloc702) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias3) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[49] gv872: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc704: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv872, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3, alloc703, gpt_neox_layers_2_mlp_dense_4h_to_h_bias3, alloc704) R.vm.kill_object(alloc703) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias3) gpt_neox_layers_2_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[43] gv873: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc705: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv873, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_2_attention_dense_q_weight3, gpt_neox_layers_2_attention_dense_q_scale3, reshape203, gpt_neox_layers_2_attention_dense_bias3, alloc704, alloc698, alloc705) R.vm.kill_object(alloc698) R.vm.kill_object(reshape203) R.vm.kill_object(alloc704) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias3) gpt_neox_layers_3_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[51] gv874: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc706: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv874, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc705, gpt_neox_layers_3_input_layernorm_weight3, gpt_neox_layers_3_input_layernorm_bias3, alloc706) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias3) gpt_neox_layers_3_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[56] gv875: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc707: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv875, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_3_attention_query_key_value_q_weight3, gpt_neox_layers_3_attention_query_key_value_q_scale3, alloc706, gpt_neox_layers_3_attention_query_key_value_bias3, alloc707) R.vm.kill_object(alloc706) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias3) gv876: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape204: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc707, gv876, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc707) gv877: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape205: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape204, gv877, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape204) gv878: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc708: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv878, R.dtype("float16")) _707: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape205, alloc708) R.vm.kill_object(reshape205) gv879: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape206: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc708, gv879, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc708) gv880: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape207: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape206, gv880, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape206) gpt_neox_layers_3_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[53] gv881: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc709: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv881, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc705, gpt_neox_layers_3_post_attention_layernorm_weight3, gpt_neox_layers_3_post_attention_layernorm_bias3, alloc709) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias3) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[62] gv882: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc710: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv882, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3, alloc709, gpt_neox_layers_3_mlp_dense_h_to_4h_bias3, alloc710) R.vm.kill_object(alloc709) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias3) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[65] gv883: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc711: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv883, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3, alloc710, gpt_neox_layers_3_mlp_dense_4h_to_h_bias3, alloc711) R.vm.kill_object(alloc710) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias3) gpt_neox_layers_3_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[59] gv884: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc712: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv884, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_3_attention_dense_q_weight3, gpt_neox_layers_3_attention_dense_q_scale3, reshape207, gpt_neox_layers_3_attention_dense_bias3, alloc711, alloc705, alloc712) R.vm.kill_object(alloc705) R.vm.kill_object(reshape207) R.vm.kill_object(alloc711) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias3) gpt_neox_layers_4_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[67] gv885: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc713: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv885, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc712, gpt_neox_layers_4_input_layernorm_weight3, gpt_neox_layers_4_input_layernorm_bias3, alloc713) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias3) gpt_neox_layers_4_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[72] gv886: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc714: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv886, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_4_attention_query_key_value_q_weight3, gpt_neox_layers_4_attention_query_key_value_q_scale3, alloc713, gpt_neox_layers_4_attention_query_key_value_bias3, alloc714) R.vm.kill_object(alloc713) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias3) gv887: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape208: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc714, gv887, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc714) gv888: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape209: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape208, gv888, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape208) gv889: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc715: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv889, R.dtype("float16")) _714: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape209, alloc715) R.vm.kill_object(reshape209) gv890: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape210: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc715, gv890, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc715) gv891: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape211: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape210, gv891, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape210) gpt_neox_layers_4_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[69] gv892: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc716: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv892, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc712, gpt_neox_layers_4_post_attention_layernorm_weight3, gpt_neox_layers_4_post_attention_layernorm_bias3, alloc716) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias3) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[78] gv893: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc717: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv893, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3, alloc716, gpt_neox_layers_4_mlp_dense_h_to_4h_bias3, alloc717) R.vm.kill_object(alloc716) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias3) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[81] gv894: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc718: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv894, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3, alloc717, gpt_neox_layers_4_mlp_dense_4h_to_h_bias3, alloc718) R.vm.kill_object(alloc717) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias3) gpt_neox_layers_4_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[75] gv895: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc719: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv895, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_4_attention_dense_q_weight3, gpt_neox_layers_4_attention_dense_q_scale3, reshape211, gpt_neox_layers_4_attention_dense_bias3, alloc718, alloc712, alloc719) R.vm.kill_object(alloc712) R.vm.kill_object(reshape211) R.vm.kill_object(alloc718) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias3) gpt_neox_layers_5_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[83] gv896: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc720: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv896, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc719, gpt_neox_layers_5_input_layernorm_weight3, gpt_neox_layers_5_input_layernorm_bias3, alloc720) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias3) gpt_neox_layers_5_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[88] gv897: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc721: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv897, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_5_attention_query_key_value_q_weight3, gpt_neox_layers_5_attention_query_key_value_q_scale3, alloc720, gpt_neox_layers_5_attention_query_key_value_bias3, alloc721) R.vm.kill_object(alloc720) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias3) gv898: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape212: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc721, gv898, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc721) gv899: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape213: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape212, gv899, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape212) gv900: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc722: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv900, R.dtype("float16")) _721: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape213, alloc722) R.vm.kill_object(reshape213) gv901: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape214: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc722, gv901, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc722) gv902: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape215: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape214, gv902, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape214) gpt_neox_layers_5_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[85] gv903: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc723: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv903, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc719, gpt_neox_layers_5_post_attention_layernorm_weight3, gpt_neox_layers_5_post_attention_layernorm_bias3, alloc723) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias3) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[94] gv904: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc724: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv904, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3, alloc723, gpt_neox_layers_5_mlp_dense_h_to_4h_bias3, alloc724) R.vm.kill_object(alloc723) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias3) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[97] gv905: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc725: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv905, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3, alloc724, gpt_neox_layers_5_mlp_dense_4h_to_h_bias3, alloc725) R.vm.kill_object(alloc724) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias3) gpt_neox_layers_5_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[91] gv906: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc726: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv906, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_5_attention_dense_q_weight3, gpt_neox_layers_5_attention_dense_q_scale3, reshape215, gpt_neox_layers_5_attention_dense_bias3, alloc725, alloc719, alloc726) R.vm.kill_object(alloc719) R.vm.kill_object(reshape215) R.vm.kill_object(alloc725) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias3) gpt_neox_layers_6_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[98] gpt_neox_layers_6_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[99] gv907: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc727: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv907, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc726, gpt_neox_layers_6_input_layernorm_weight3, gpt_neox_layers_6_input_layernorm_bias3, alloc727) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_bias3) gpt_neox_layers_6_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[102] gpt_neox_layers_6_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[103] gpt_neox_layers_6_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[104] gv908: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc728: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv908, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_6_attention_query_key_value_q_weight3, gpt_neox_layers_6_attention_query_key_value_q_scale3, alloc727, gpt_neox_layers_6_attention_query_key_value_bias3, alloc728) R.vm.kill_object(alloc727) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_bias3) gv909: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape216: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc728, gv909, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc728) gv910: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape217: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape216, gv910, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape216) gv911: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc729: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv911, R.dtype("float16")) _728: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape217, alloc729) R.vm.kill_object(reshape217) gv912: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape218: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc729, gv912, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc729) gv913: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape219: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape218, gv913, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape218) gpt_neox_layers_6_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[100] gpt_neox_layers_6_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[101] gv914: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc730: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv914, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc726, gpt_neox_layers_6_post_attention_layernorm_weight3, gpt_neox_layers_6_post_attention_layernorm_bias3, alloc730) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_bias3) gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[108] gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[109] gpt_neox_layers_6_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[110] gv915: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc731: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv915, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale3, alloc730, gpt_neox_layers_6_mlp_dense_h_to_4h_bias3, alloc731) R.vm.kill_object(alloc730) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_bias3) gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[111] gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[112] gpt_neox_layers_6_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[113] gv916: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc732: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv916, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale3, alloc731, gpt_neox_layers_6_mlp_dense_4h_to_h_bias3, alloc732) R.vm.kill_object(alloc731) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_bias3) gpt_neox_layers_6_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[105] gpt_neox_layers_6_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[106] gpt_neox_layers_6_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[107] gv917: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc733: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv917, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_6_attention_dense_q_weight3, gpt_neox_layers_6_attention_dense_q_scale3, reshape219, gpt_neox_layers_6_attention_dense_bias3, alloc732, alloc726, alloc733) R.vm.kill_object(alloc726) R.vm.kill_object(reshape219) R.vm.kill_object(alloc732) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_6_attention_dense_bias3) gpt_neox_layers_7_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[114] gpt_neox_layers_7_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[115] gv918: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc734: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv918, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc733, gpt_neox_layers_7_input_layernorm_weight3, gpt_neox_layers_7_input_layernorm_bias3, alloc734) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_bias3) gpt_neox_layers_7_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[118] gpt_neox_layers_7_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[119] gpt_neox_layers_7_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[120] gv919: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc735: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv919, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_7_attention_query_key_value_q_weight3, gpt_neox_layers_7_attention_query_key_value_q_scale3, alloc734, gpt_neox_layers_7_attention_query_key_value_bias3, alloc735) R.vm.kill_object(alloc734) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_bias3) gv920: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape220: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc735, gv920, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc735) gv921: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape221: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape220, gv921, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape220) gv922: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc736: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv922, R.dtype("float16")) _735: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape221, alloc736) R.vm.kill_object(reshape221) gv923: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape222: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc736, gv923, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc736) gv924: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape223: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape222, gv924, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape222) gpt_neox_layers_7_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[116] gpt_neox_layers_7_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[117] gv925: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc737: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv925, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc733, gpt_neox_layers_7_post_attention_layernorm_weight3, gpt_neox_layers_7_post_attention_layernorm_bias3, alloc737) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_bias3) gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[124] gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[125] gpt_neox_layers_7_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[126] gv926: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc738: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv926, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale3, alloc737, gpt_neox_layers_7_mlp_dense_h_to_4h_bias3, alloc738) R.vm.kill_object(alloc737) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_bias3) gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[127] gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[128] gpt_neox_layers_7_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[129] gv927: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc739: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv927, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale3, alloc738, gpt_neox_layers_7_mlp_dense_4h_to_h_bias3, alloc739) R.vm.kill_object(alloc738) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_bias3) gpt_neox_layers_7_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[121] gpt_neox_layers_7_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[122] gpt_neox_layers_7_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[123] gv928: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc740: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv928, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_7_attention_dense_q_weight3, gpt_neox_layers_7_attention_dense_q_scale3, reshape223, gpt_neox_layers_7_attention_dense_bias3, alloc739, alloc733, alloc740) R.vm.kill_object(alloc733) R.vm.kill_object(reshape223) R.vm.kill_object(alloc739) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_7_attention_dense_bias3) gpt_neox_layers_8_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[130] gpt_neox_layers_8_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[131] gv929: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc741: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv929, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc740, gpt_neox_layers_8_input_layernorm_weight3, gpt_neox_layers_8_input_layernorm_bias3, alloc741) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_bias3) gpt_neox_layers_8_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[134] gpt_neox_layers_8_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[135] gpt_neox_layers_8_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[136] gv930: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc742: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv930, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_8_attention_query_key_value_q_weight3, gpt_neox_layers_8_attention_query_key_value_q_scale3, alloc741, gpt_neox_layers_8_attention_query_key_value_bias3, alloc742) R.vm.kill_object(alloc741) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_bias3) gv931: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape224: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc742, gv931, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc742) gv932: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape225: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape224, gv932, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape224) gv933: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc743: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv933, R.dtype("float16")) _742: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape225, alloc743) R.vm.kill_object(reshape225) gv934: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape226: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc743, gv934, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc743) gv935: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape227: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape226, gv935, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape226) gpt_neox_layers_8_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[132] gpt_neox_layers_8_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[133] gv936: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc744: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv936, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc740, gpt_neox_layers_8_post_attention_layernorm_weight3, gpt_neox_layers_8_post_attention_layernorm_bias3, alloc744) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_bias3) gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[140] gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[141] gpt_neox_layers_8_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[142] gv937: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc745: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv937, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale3, alloc744, gpt_neox_layers_8_mlp_dense_h_to_4h_bias3, alloc745) R.vm.kill_object(alloc744) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_bias3) gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[143] gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[144] gpt_neox_layers_8_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[145] gv938: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc746: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv938, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale3, alloc745, gpt_neox_layers_8_mlp_dense_4h_to_h_bias3, alloc746) R.vm.kill_object(alloc745) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_bias3) gpt_neox_layers_8_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[137] gpt_neox_layers_8_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[138] gpt_neox_layers_8_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[139] gv939: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc747: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv939, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_8_attention_dense_q_weight3, gpt_neox_layers_8_attention_dense_q_scale3, reshape227, gpt_neox_layers_8_attention_dense_bias3, alloc746, alloc740, alloc747) R.vm.kill_object(alloc740) R.vm.kill_object(reshape227) R.vm.kill_object(alloc746) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_8_attention_dense_bias3) gpt_neox_layers_9_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[146] gpt_neox_layers_9_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[147] gv940: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc748: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv940, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc747, gpt_neox_layers_9_input_layernorm_weight3, gpt_neox_layers_9_input_layernorm_bias3, alloc748) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_bias3) gpt_neox_layers_9_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[150] gpt_neox_layers_9_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[151] gpt_neox_layers_9_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[152] gv941: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc749: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv941, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_9_attention_query_key_value_q_weight3, gpt_neox_layers_9_attention_query_key_value_q_scale3, alloc748, gpt_neox_layers_9_attention_query_key_value_bias3, alloc749) R.vm.kill_object(alloc748) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_bias3) gv942: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape228: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc749, gv942, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc749) gv943: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape229: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape228, gv943, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape228) gv944: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc750: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv944, R.dtype("float16")) _749: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape229, alloc750) R.vm.kill_object(reshape229) gv945: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape230: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc750, gv945, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc750) gv946: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape231: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape230, gv946, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape230) gpt_neox_layers_9_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[148] gpt_neox_layers_9_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[149] gv947: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc751: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv947, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc747, gpt_neox_layers_9_post_attention_layernorm_weight3, gpt_neox_layers_9_post_attention_layernorm_bias3, alloc751) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_bias3) gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[156] gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[157] gpt_neox_layers_9_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[158] gv948: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc752: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv948, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale3, alloc751, gpt_neox_layers_9_mlp_dense_h_to_4h_bias3, alloc752) R.vm.kill_object(alloc751) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_bias3) gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[159] gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[160] gpt_neox_layers_9_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[161] gv949: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc753: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv949, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale3, alloc752, gpt_neox_layers_9_mlp_dense_4h_to_h_bias3, alloc753) R.vm.kill_object(alloc752) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_bias3) gpt_neox_layers_9_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[153] gpt_neox_layers_9_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[154] gpt_neox_layers_9_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[155] gv950: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc754: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv950, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_9_attention_dense_q_weight3, gpt_neox_layers_9_attention_dense_q_scale3, reshape231, gpt_neox_layers_9_attention_dense_bias3, alloc753, alloc747, alloc754) R.vm.kill_object(alloc747) R.vm.kill_object(reshape231) R.vm.kill_object(alloc753) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_9_attention_dense_bias3) gpt_neox_layers_10_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[162] gpt_neox_layers_10_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[163] gv951: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc755: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv951, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc754, gpt_neox_layers_10_input_layernorm_weight3, gpt_neox_layers_10_input_layernorm_bias3, alloc755) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_bias3) gpt_neox_layers_10_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[166] gpt_neox_layers_10_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[167] gpt_neox_layers_10_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[168] gv952: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc756: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv952, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_10_attention_query_key_value_q_weight3, gpt_neox_layers_10_attention_query_key_value_q_scale3, alloc755, gpt_neox_layers_10_attention_query_key_value_bias3, alloc756) R.vm.kill_object(alloc755) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_bias3) gv953: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape232: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc756, gv953, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc756) gv954: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape233: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape232, gv954, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape232) gv955: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc757: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv955, R.dtype("float16")) _756: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape233, alloc757) R.vm.kill_object(reshape233) gv956: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape234: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc757, gv956, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc757) gv957: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape235: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape234, gv957, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape234) gpt_neox_layers_10_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[164] gpt_neox_layers_10_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[165] gv958: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc758: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv958, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc754, gpt_neox_layers_10_post_attention_layernorm_weight3, gpt_neox_layers_10_post_attention_layernorm_bias3, alloc758) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_bias3) gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[172] gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[173] gpt_neox_layers_10_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[174] gv959: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc759: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv959, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale3, alloc758, gpt_neox_layers_10_mlp_dense_h_to_4h_bias3, alloc759) R.vm.kill_object(alloc758) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_bias3) gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[175] gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[176] gpt_neox_layers_10_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[177] gv960: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc760: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv960, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale3, alloc759, gpt_neox_layers_10_mlp_dense_4h_to_h_bias3, alloc760) R.vm.kill_object(alloc759) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_bias3) gpt_neox_layers_10_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[169] gpt_neox_layers_10_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[170] gpt_neox_layers_10_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[171] gv961: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc761: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv961, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_10_attention_dense_q_weight3, gpt_neox_layers_10_attention_dense_q_scale3, reshape235, gpt_neox_layers_10_attention_dense_bias3, alloc760, alloc754, alloc761) R.vm.kill_object(alloc754) R.vm.kill_object(reshape235) R.vm.kill_object(alloc760) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_10_attention_dense_bias3) gpt_neox_layers_11_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[178] gpt_neox_layers_11_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[179] gv962: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc762: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv962, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc761, gpt_neox_layers_11_input_layernorm_weight3, gpt_neox_layers_11_input_layernorm_bias3, alloc762) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_bias3) gpt_neox_layers_11_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[182] gpt_neox_layers_11_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[183] gpt_neox_layers_11_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[184] gv963: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc763: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv963, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_11_attention_query_key_value_q_weight3, gpt_neox_layers_11_attention_query_key_value_q_scale3, alloc762, gpt_neox_layers_11_attention_query_key_value_bias3, alloc763) R.vm.kill_object(alloc762) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_bias3) gv964: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape236: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc763, gv964, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc763) gv965: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape237: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape236, gv965, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape236) gv966: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc764: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv966, R.dtype("float16")) _763: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape237, alloc764) R.vm.kill_object(reshape237) gv967: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape238: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc764, gv967, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc764) gv968: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape239: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape238, gv968, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape238) gpt_neox_layers_11_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[180] gpt_neox_layers_11_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[181] gv969: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc765: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv969, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc761, gpt_neox_layers_11_post_attention_layernorm_weight3, gpt_neox_layers_11_post_attention_layernorm_bias3, alloc765) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_bias3) gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[188] gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[189] gpt_neox_layers_11_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[190] gv970: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc766: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv970, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale3, alloc765, gpt_neox_layers_11_mlp_dense_h_to_4h_bias3, alloc766) R.vm.kill_object(alloc765) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_bias3) gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[191] gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[192] gpt_neox_layers_11_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[193] gv971: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc767: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv971, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale3, alloc766, gpt_neox_layers_11_mlp_dense_4h_to_h_bias3, alloc767) R.vm.kill_object(alloc766) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_bias3) gpt_neox_layers_11_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[185] gpt_neox_layers_11_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[186] gpt_neox_layers_11_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[187] gv972: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc768: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv972, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_11_attention_dense_q_weight3, gpt_neox_layers_11_attention_dense_q_scale3, reshape239, gpt_neox_layers_11_attention_dense_bias3, alloc767, alloc761, alloc768) R.vm.kill_object(alloc761) R.vm.kill_object(reshape239) R.vm.kill_object(alloc767) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_11_attention_dense_bias3) gpt_neox_layers_12_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[194] gpt_neox_layers_12_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[195] gv973: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc769: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv973, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc768, gpt_neox_layers_12_input_layernorm_weight3, gpt_neox_layers_12_input_layernorm_bias3, alloc769) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_bias3) gpt_neox_layers_12_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[198] gpt_neox_layers_12_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[199] gpt_neox_layers_12_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[200] gv974: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc770: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv974, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_12_attention_query_key_value_q_weight3, gpt_neox_layers_12_attention_query_key_value_q_scale3, alloc769, gpt_neox_layers_12_attention_query_key_value_bias3, alloc770) R.vm.kill_object(alloc769) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_bias3) gv975: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape240: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc770, gv975, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc770) gv976: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape241: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape240, gv976, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape240) gv977: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc771: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv977, R.dtype("float16")) _770: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape241, alloc771) R.vm.kill_object(reshape241) gv978: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape242: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc771, gv978, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc771) gv979: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape243: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape242, gv979, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape242) gpt_neox_layers_12_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[196] gpt_neox_layers_12_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[197] gv980: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc772: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv980, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc768, gpt_neox_layers_12_post_attention_layernorm_weight3, gpt_neox_layers_12_post_attention_layernorm_bias3, alloc772) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_bias3) gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[204] gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[205] gpt_neox_layers_12_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[206] gv981: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc773: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv981, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale3, alloc772, gpt_neox_layers_12_mlp_dense_h_to_4h_bias3, alloc773) R.vm.kill_object(alloc772) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_bias3) gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[207] gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[208] gpt_neox_layers_12_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[209] gv982: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc774: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv982, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale3, alloc773, gpt_neox_layers_12_mlp_dense_4h_to_h_bias3, alloc774) R.vm.kill_object(alloc773) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_bias3) gpt_neox_layers_12_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[201] gpt_neox_layers_12_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[202] gpt_neox_layers_12_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[203] gv983: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc775: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv983, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_12_attention_dense_q_weight3, gpt_neox_layers_12_attention_dense_q_scale3, reshape243, gpt_neox_layers_12_attention_dense_bias3, alloc774, alloc768, alloc775) R.vm.kill_object(alloc768) R.vm.kill_object(reshape243) R.vm.kill_object(alloc774) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_12_attention_dense_bias3) gpt_neox_layers_13_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[210] gpt_neox_layers_13_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[211] gv984: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc776: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv984, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc775, gpt_neox_layers_13_input_layernorm_weight3, gpt_neox_layers_13_input_layernorm_bias3, alloc776) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_bias3) gpt_neox_layers_13_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[214] gpt_neox_layers_13_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[215] gpt_neox_layers_13_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[216] gv985: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc777: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv985, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_13_attention_query_key_value_q_weight3, gpt_neox_layers_13_attention_query_key_value_q_scale3, alloc776, gpt_neox_layers_13_attention_query_key_value_bias3, alloc777) R.vm.kill_object(alloc776) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_bias3) gv986: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape244: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc777, gv986, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc777) gv987: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape245: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape244, gv987, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape244) gv988: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc778: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv988, R.dtype("float16")) _777: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape245, alloc778) R.vm.kill_object(reshape245) gv989: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape246: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc778, gv989, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc778) gv990: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape247: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape246, gv990, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape246) gpt_neox_layers_13_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[212] gpt_neox_layers_13_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[213] gv991: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc779: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv991, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc775, gpt_neox_layers_13_post_attention_layernorm_weight3, gpt_neox_layers_13_post_attention_layernorm_bias3, alloc779) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_bias3) gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[220] gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[221] gpt_neox_layers_13_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[222] gv992: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc780: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv992, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale3, alloc779, gpt_neox_layers_13_mlp_dense_h_to_4h_bias3, alloc780) R.vm.kill_object(alloc779) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_bias3) gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[223] gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[224] gpt_neox_layers_13_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[225] gv993: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc781: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv993, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale3, alloc780, gpt_neox_layers_13_mlp_dense_4h_to_h_bias3, alloc781) R.vm.kill_object(alloc780) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_bias3) gpt_neox_layers_13_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[217] gpt_neox_layers_13_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[218] gpt_neox_layers_13_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[219] gv994: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc782: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv994, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_13_attention_dense_q_weight3, gpt_neox_layers_13_attention_dense_q_scale3, reshape247, gpt_neox_layers_13_attention_dense_bias3, alloc781, alloc775, alloc782) R.vm.kill_object(alloc775) R.vm.kill_object(reshape247) R.vm.kill_object(alloc781) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_13_attention_dense_bias3) gpt_neox_layers_14_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[226] gpt_neox_layers_14_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[227] gv995: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc783: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv995, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc782, gpt_neox_layers_14_input_layernorm_weight3, gpt_neox_layers_14_input_layernorm_bias3, alloc783) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_bias3) gpt_neox_layers_14_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[230] gpt_neox_layers_14_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[231] gpt_neox_layers_14_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[232] gv996: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc784: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv996, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_14_attention_query_key_value_q_weight3, gpt_neox_layers_14_attention_query_key_value_q_scale3, alloc783, gpt_neox_layers_14_attention_query_key_value_bias3, alloc784) R.vm.kill_object(alloc783) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_bias3) gv997: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape248: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc784, gv997, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc784) gv998: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape249: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape248, gv998, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape248) gv999: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc785: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv999, R.dtype("float16")) _784: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape249, alloc785) R.vm.kill_object(reshape249) gv1000: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape250: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc785, gv1000, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc785) gv1001: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape251: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape250, gv1001, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape250) gpt_neox_layers_14_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[228] gpt_neox_layers_14_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[229] gv1002: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc786: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1002, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc782, gpt_neox_layers_14_post_attention_layernorm_weight3, gpt_neox_layers_14_post_attention_layernorm_bias3, alloc786) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_bias3) gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[236] gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[237] gpt_neox_layers_14_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[238] gv1003: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc787: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1003, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale3, alloc786, gpt_neox_layers_14_mlp_dense_h_to_4h_bias3, alloc787) R.vm.kill_object(alloc786) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_bias3) gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[239] gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[240] gpt_neox_layers_14_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[241] gv1004: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc788: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1004, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale3, alloc787, gpt_neox_layers_14_mlp_dense_4h_to_h_bias3, alloc788) R.vm.kill_object(alloc787) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_bias3) gpt_neox_layers_14_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[233] gpt_neox_layers_14_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[234] gpt_neox_layers_14_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[235] gv1005: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc789: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1005, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_14_attention_dense_q_weight3, gpt_neox_layers_14_attention_dense_q_scale3, reshape251, gpt_neox_layers_14_attention_dense_bias3, alloc788, alloc782, alloc789) R.vm.kill_object(alloc782) R.vm.kill_object(reshape251) R.vm.kill_object(alloc788) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_14_attention_dense_bias3) gpt_neox_layers_15_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[242] gpt_neox_layers_15_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[243] gv1006: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc790: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1006, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc789, gpt_neox_layers_15_input_layernorm_weight3, gpt_neox_layers_15_input_layernorm_bias3, alloc790) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_bias3) gpt_neox_layers_15_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[246] gpt_neox_layers_15_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[247] gpt_neox_layers_15_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[248] gv1007: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc791: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1007, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_15_attention_query_key_value_q_weight3, gpt_neox_layers_15_attention_query_key_value_q_scale3, alloc790, gpt_neox_layers_15_attention_query_key_value_bias3, alloc791) R.vm.kill_object(alloc790) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_bias3) gv1008: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape252: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc791, gv1008, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc791) gv1009: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape253: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape252, gv1009, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape252) gv1010: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc792: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1010, R.dtype("float16")) _791: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape253, alloc792) R.vm.kill_object(reshape253) gv1011: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape254: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc792, gv1011, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc792) gv1012: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape255: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape254, gv1012, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape254) gpt_neox_layers_15_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[244] gpt_neox_layers_15_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[245] gv1013: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc793: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1013, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc789, gpt_neox_layers_15_post_attention_layernorm_weight3, gpt_neox_layers_15_post_attention_layernorm_bias3, alloc793) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_bias3) gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[252] gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[253] gpt_neox_layers_15_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[254] gv1014: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc794: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1014, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale3, alloc793, gpt_neox_layers_15_mlp_dense_h_to_4h_bias3, alloc794) R.vm.kill_object(alloc793) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_bias3) gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[255] gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[256] gpt_neox_layers_15_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[257] gv1015: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc795: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1015, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale3, alloc794, gpt_neox_layers_15_mlp_dense_4h_to_h_bias3, alloc795) R.vm.kill_object(alloc794) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_bias3) gpt_neox_layers_15_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[249] gpt_neox_layers_15_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[250] gpt_neox_layers_15_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[251] gv1016: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc796: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1016, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_15_attention_dense_q_weight3, gpt_neox_layers_15_attention_dense_q_scale3, reshape255, gpt_neox_layers_15_attention_dense_bias3, alloc795, alloc789, alloc796) R.vm.kill_object(alloc789) R.vm.kill_object(reshape255) R.vm.kill_object(alloc795) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_15_attention_dense_bias3) gpt_neox_layers_16_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[258] gpt_neox_layers_16_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[259] gv1017: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc797: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1017, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc796, gpt_neox_layers_16_input_layernorm_weight3, gpt_neox_layers_16_input_layernorm_bias3, alloc797) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_bias3) gpt_neox_layers_16_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[262] gpt_neox_layers_16_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[263] gpt_neox_layers_16_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[264] gv1018: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc798: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1018, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_16_attention_query_key_value_q_weight3, gpt_neox_layers_16_attention_query_key_value_q_scale3, alloc797, gpt_neox_layers_16_attention_query_key_value_bias3, alloc798) R.vm.kill_object(alloc797) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_bias3) gv1019: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape256: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc798, gv1019, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc798) gv1020: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape257: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape256, gv1020, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape256) gv1021: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc799: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1021, R.dtype("float16")) _798: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape257, alloc799) R.vm.kill_object(reshape257) gv1022: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape258: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc799, gv1022, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc799) gv1023: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape259: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape258, gv1023, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape258) gpt_neox_layers_16_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[260] gpt_neox_layers_16_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[261] gv1024: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc800: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1024, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc796, gpt_neox_layers_16_post_attention_layernorm_weight3, gpt_neox_layers_16_post_attention_layernorm_bias3, alloc800) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_bias3) gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[268] gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[269] gpt_neox_layers_16_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[270] gv1025: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc801: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1025, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale3, alloc800, gpt_neox_layers_16_mlp_dense_h_to_4h_bias3, alloc801) R.vm.kill_object(alloc800) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_bias3) gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[271] gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[272] gpt_neox_layers_16_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[273] gv1026: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc802: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1026, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale3, alloc801, gpt_neox_layers_16_mlp_dense_4h_to_h_bias3, alloc802) R.vm.kill_object(alloc801) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_bias3) gpt_neox_layers_16_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[265] gpt_neox_layers_16_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[266] gpt_neox_layers_16_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[267] gv1027: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc803: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1027, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_16_attention_dense_q_weight3, gpt_neox_layers_16_attention_dense_q_scale3, reshape259, gpt_neox_layers_16_attention_dense_bias3, alloc802, alloc796, alloc803) R.vm.kill_object(alloc796) R.vm.kill_object(reshape259) R.vm.kill_object(alloc802) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_16_attention_dense_bias3) gpt_neox_layers_17_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[274] gpt_neox_layers_17_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[275] gv1028: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc804: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1028, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc803, gpt_neox_layers_17_input_layernorm_weight3, gpt_neox_layers_17_input_layernorm_bias3, alloc804) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_bias3) gpt_neox_layers_17_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[278] gpt_neox_layers_17_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[279] gpt_neox_layers_17_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[280] gv1029: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc805: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1029, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_17_attention_query_key_value_q_weight3, gpt_neox_layers_17_attention_query_key_value_q_scale3, alloc804, gpt_neox_layers_17_attention_query_key_value_bias3, alloc805) R.vm.kill_object(alloc804) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_bias3) gv1030: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape260: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc805, gv1030, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc805) gv1031: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape261: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape260, gv1031, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape260) gv1032: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc806: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1032, R.dtype("float16")) _805: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape261, alloc806) R.vm.kill_object(reshape261) gv1033: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape262: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc806, gv1033, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc806) gv1034: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape263: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape262, gv1034, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape262) gpt_neox_layers_17_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[276] gpt_neox_layers_17_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[277] gv1035: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc807: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1035, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc803, gpt_neox_layers_17_post_attention_layernorm_weight3, gpt_neox_layers_17_post_attention_layernorm_bias3, alloc807) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_bias3) gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[284] gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[285] gpt_neox_layers_17_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[286] gv1036: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc808: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1036, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale3, alloc807, gpt_neox_layers_17_mlp_dense_h_to_4h_bias3, alloc808) R.vm.kill_object(alloc807) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_bias3) gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[287] gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[288] gpt_neox_layers_17_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[289] gv1037: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc809: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1037, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale3, alloc808, gpt_neox_layers_17_mlp_dense_4h_to_h_bias3, alloc809) R.vm.kill_object(alloc808) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_bias3) gpt_neox_layers_17_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[281] gpt_neox_layers_17_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[282] gpt_neox_layers_17_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[283] gv1038: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc810: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1038, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_17_attention_dense_q_weight3, gpt_neox_layers_17_attention_dense_q_scale3, reshape263, gpt_neox_layers_17_attention_dense_bias3, alloc809, alloc803, alloc810) R.vm.kill_object(alloc803) R.vm.kill_object(reshape263) R.vm.kill_object(alloc809) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_17_attention_dense_bias3) gpt_neox_layers_18_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[290] gpt_neox_layers_18_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[291] gv1039: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc811: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1039, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc810, gpt_neox_layers_18_input_layernorm_weight3, gpt_neox_layers_18_input_layernorm_bias3, alloc811) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_bias3) gpt_neox_layers_18_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[294] gpt_neox_layers_18_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[295] gpt_neox_layers_18_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[296] gv1040: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc812: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1040, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_18_attention_query_key_value_q_weight3, gpt_neox_layers_18_attention_query_key_value_q_scale3, alloc811, gpt_neox_layers_18_attention_query_key_value_bias3, alloc812) R.vm.kill_object(alloc811) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_bias3) gv1041: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape264: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc812, gv1041, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc812) gv1042: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape265: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape264, gv1042, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape264) gv1043: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc813: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1043, R.dtype("float16")) _812: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape265, alloc813) R.vm.kill_object(reshape265) gv1044: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape266: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc813, gv1044, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc813) gv1045: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape267: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape266, gv1045, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape266) gpt_neox_layers_18_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[292] gpt_neox_layers_18_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[293] gv1046: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc814: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1046, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc810, gpt_neox_layers_18_post_attention_layernorm_weight3, gpt_neox_layers_18_post_attention_layernorm_bias3, alloc814) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_bias3) gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[300] gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[301] gpt_neox_layers_18_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[302] gv1047: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc815: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1047, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale3, alloc814, gpt_neox_layers_18_mlp_dense_h_to_4h_bias3, alloc815) R.vm.kill_object(alloc814) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_bias3) gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[303] gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[304] gpt_neox_layers_18_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[305] gv1048: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc816: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1048, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale3, alloc815, gpt_neox_layers_18_mlp_dense_4h_to_h_bias3, alloc816) R.vm.kill_object(alloc815) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_bias3) gpt_neox_layers_18_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[297] gpt_neox_layers_18_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[298] gpt_neox_layers_18_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[299] gv1049: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc817: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1049, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_18_attention_dense_q_weight3, gpt_neox_layers_18_attention_dense_q_scale3, reshape267, gpt_neox_layers_18_attention_dense_bias3, alloc816, alloc810, alloc817) R.vm.kill_object(alloc810) R.vm.kill_object(reshape267) R.vm.kill_object(alloc816) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_18_attention_dense_bias3) gpt_neox_layers_19_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[306] gpt_neox_layers_19_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[307] gv1050: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc818: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1050, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc817, gpt_neox_layers_19_input_layernorm_weight3, gpt_neox_layers_19_input_layernorm_bias3, alloc818) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_bias3) gpt_neox_layers_19_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[310] gpt_neox_layers_19_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[311] gpt_neox_layers_19_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[312] gv1051: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc819: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1051, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_19_attention_query_key_value_q_weight3, gpt_neox_layers_19_attention_query_key_value_q_scale3, alloc818, gpt_neox_layers_19_attention_query_key_value_bias3, alloc819) R.vm.kill_object(alloc818) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_bias3) gv1052: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape268: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc819, gv1052, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc819) gv1053: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape269: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape268, gv1053, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape268) gv1054: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc820: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1054, R.dtype("float16")) _819: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape269, alloc820) R.vm.kill_object(reshape269) gv1055: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape270: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc820, gv1055, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc820) gv1056: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape271: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape270, gv1056, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape270) gpt_neox_layers_19_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[308] gpt_neox_layers_19_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[309] gv1057: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc821: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1057, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc817, gpt_neox_layers_19_post_attention_layernorm_weight3, gpt_neox_layers_19_post_attention_layernorm_bias3, alloc821) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_bias3) gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[316] gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[317] gpt_neox_layers_19_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[318] gv1058: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc822: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1058, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale3, alloc821, gpt_neox_layers_19_mlp_dense_h_to_4h_bias3, alloc822) R.vm.kill_object(alloc821) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_bias3) gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[319] gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[320] gpt_neox_layers_19_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[321] gv1059: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc823: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1059, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale3, alloc822, gpt_neox_layers_19_mlp_dense_4h_to_h_bias3, alloc823) R.vm.kill_object(alloc822) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_bias3) gpt_neox_layers_19_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[313] gpt_neox_layers_19_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[314] gpt_neox_layers_19_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[315] gv1060: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc824: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1060, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_19_attention_dense_q_weight3, gpt_neox_layers_19_attention_dense_q_scale3, reshape271, gpt_neox_layers_19_attention_dense_bias3, alloc823, alloc817, alloc824) R.vm.kill_object(alloc817) R.vm.kill_object(reshape271) R.vm.kill_object(alloc823) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_19_attention_dense_bias3) gpt_neox_layers_20_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[322] gpt_neox_layers_20_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[323] gv1061: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc825: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1061, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc824, gpt_neox_layers_20_input_layernorm_weight3, gpt_neox_layers_20_input_layernorm_bias3, alloc825) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_bias3) gpt_neox_layers_20_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[326] gpt_neox_layers_20_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[327] gpt_neox_layers_20_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[328] gv1062: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc826: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1062, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_20_attention_query_key_value_q_weight3, gpt_neox_layers_20_attention_query_key_value_q_scale3, alloc825, gpt_neox_layers_20_attention_query_key_value_bias3, alloc826) R.vm.kill_object(alloc825) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_bias3) gv1063: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape272: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc826, gv1063, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc826) gv1064: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape273: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape272, gv1064, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape272) gv1065: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc827: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1065, R.dtype("float16")) _826: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape273, alloc827) R.vm.kill_object(reshape273) gv1066: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape274: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc827, gv1066, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc827) gv1067: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape275: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape274, gv1067, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape274) gpt_neox_layers_20_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[324] gpt_neox_layers_20_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[325] gv1068: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc828: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1068, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc824, gpt_neox_layers_20_post_attention_layernorm_weight3, gpt_neox_layers_20_post_attention_layernorm_bias3, alloc828) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_bias3) gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[332] gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[333] gpt_neox_layers_20_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[334] gv1069: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc829: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1069, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale3, alloc828, gpt_neox_layers_20_mlp_dense_h_to_4h_bias3, alloc829) R.vm.kill_object(alloc828) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_bias3) gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[335] gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[336] gpt_neox_layers_20_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[337] gv1070: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc830: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1070, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale3, alloc829, gpt_neox_layers_20_mlp_dense_4h_to_h_bias3, alloc830) R.vm.kill_object(alloc829) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_bias3) gpt_neox_layers_20_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[329] gpt_neox_layers_20_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[330] gpt_neox_layers_20_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[331] gv1071: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc831: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1071, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_20_attention_dense_q_weight3, gpt_neox_layers_20_attention_dense_q_scale3, reshape275, gpt_neox_layers_20_attention_dense_bias3, alloc830, alloc824, alloc831) R.vm.kill_object(alloc824) R.vm.kill_object(reshape275) R.vm.kill_object(alloc830) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_20_attention_dense_bias3) gpt_neox_layers_21_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[338] gpt_neox_layers_21_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[339] gv1072: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc832: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1072, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc831, gpt_neox_layers_21_input_layernorm_weight3, gpt_neox_layers_21_input_layernorm_bias3, alloc832) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_bias3) gpt_neox_layers_21_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[342] gpt_neox_layers_21_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[343] gpt_neox_layers_21_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[344] gv1073: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc833: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1073, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_21_attention_query_key_value_q_weight3, gpt_neox_layers_21_attention_query_key_value_q_scale3, alloc832, gpt_neox_layers_21_attention_query_key_value_bias3, alloc833) R.vm.kill_object(alloc832) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_bias3) gv1074: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape276: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc833, gv1074, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc833) gv1075: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape277: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape276, gv1075, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape276) gv1076: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc834: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1076, R.dtype("float16")) _833: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape277, alloc834) R.vm.kill_object(reshape277) gv1077: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape278: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc834, gv1077, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc834) gv1078: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape279: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape278, gv1078, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape278) gpt_neox_layers_21_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[340] gpt_neox_layers_21_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[341] gv1079: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc835: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1079, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc831, gpt_neox_layers_21_post_attention_layernorm_weight3, gpt_neox_layers_21_post_attention_layernorm_bias3, alloc835) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_bias3) gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[348] gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[349] gpt_neox_layers_21_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[350] gv1080: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc836: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1080, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale3, alloc835, gpt_neox_layers_21_mlp_dense_h_to_4h_bias3, alloc836) R.vm.kill_object(alloc835) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_bias3) gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[351] gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[352] gpt_neox_layers_21_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[353] gv1081: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc837: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1081, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale3, alloc836, gpt_neox_layers_21_mlp_dense_4h_to_h_bias3, alloc837) R.vm.kill_object(alloc836) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_bias3) gpt_neox_layers_21_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[345] gpt_neox_layers_21_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[346] gpt_neox_layers_21_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[347] gv1082: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc838: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1082, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_21_attention_dense_q_weight3, gpt_neox_layers_21_attention_dense_q_scale3, reshape279, gpt_neox_layers_21_attention_dense_bias3, alloc837, alloc831, alloc838) R.vm.kill_object(alloc831) R.vm.kill_object(reshape279) R.vm.kill_object(alloc837) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_21_attention_dense_bias3) gpt_neox_layers_22_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[354] gpt_neox_layers_22_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[355] gv1083: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc839: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1083, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc838, gpt_neox_layers_22_input_layernorm_weight3, gpt_neox_layers_22_input_layernorm_bias3, alloc839) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_bias3) gpt_neox_layers_22_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[358] gpt_neox_layers_22_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[359] gpt_neox_layers_22_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[360] gv1084: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc840: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1084, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_22_attention_query_key_value_q_weight3, gpt_neox_layers_22_attention_query_key_value_q_scale3, alloc839, gpt_neox_layers_22_attention_query_key_value_bias3, alloc840) R.vm.kill_object(alloc839) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_bias3) gv1085: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape280: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc840, gv1085, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc840) gv1086: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape281: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape280, gv1086, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape280) gv1087: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc841: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1087, R.dtype("float16")) _840: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape281, alloc841) R.vm.kill_object(reshape281) gv1088: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape282: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc841, gv1088, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc841) gv1089: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape283: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape282, gv1089, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape282) gpt_neox_layers_22_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[356] gpt_neox_layers_22_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[357] gv1090: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc842: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1090, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc838, gpt_neox_layers_22_post_attention_layernorm_weight3, gpt_neox_layers_22_post_attention_layernorm_bias3, alloc842) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_bias3) gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[364] gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[365] gpt_neox_layers_22_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[366] gv1091: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc843: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1091, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale3, alloc842, gpt_neox_layers_22_mlp_dense_h_to_4h_bias3, alloc843) R.vm.kill_object(alloc842) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_bias3) gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[367] gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[368] gpt_neox_layers_22_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[369] gv1092: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc844: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1092, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale3, alloc843, gpt_neox_layers_22_mlp_dense_4h_to_h_bias3, alloc844) R.vm.kill_object(alloc843) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_bias3) gpt_neox_layers_22_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[361] gpt_neox_layers_22_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[362] gpt_neox_layers_22_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[363] gv1093: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc845: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1093, R.dtype("float16")) R.vm.kill_object(storage27) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_22_attention_dense_q_weight3, gpt_neox_layers_22_attention_dense_q_scale3, reshape283, gpt_neox_layers_22_attention_dense_bias3, alloc844, alloc838, alloc845) R.vm.kill_object(alloc838) R.vm.kill_object(reshape283) R.vm.kill_object(alloc844) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_22_attention_dense_bias3) gpt_neox_layers_23_input_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[370] gpt_neox_layers_23_input_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[371] gv1094: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc846: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1094, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc845, gpt_neox_layers_23_input_layernorm_weight3, gpt_neox_layers_23_input_layernorm_bias3, alloc846) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_bias3) gpt_neox_layers_23_attention_query_key_value_q_weight3: R.Tensor((3072, 128), dtype="uint32") = packed_params[374] gpt_neox_layers_23_attention_query_key_value_q_scale3: R.Tensor((3072, 32), dtype="float16") = packed_params[375] gpt_neox_layers_23_attention_query_key_value_bias3: R.Tensor((3072,), dtype="float16") = packed_params[376] gv1095: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc847: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1095, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_23_attention_query_key_value_q_weight3, gpt_neox_layers_23_attention_query_key_value_q_scale3, alloc846, gpt_neox_layers_23_attention_query_key_value_bias3, alloc847) R.vm.kill_object(alloc846) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_weight3) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_scale3) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_bias3) gv1096: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape284: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc847, gv1096, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc847) gv1097: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape285: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape284, gv1097, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape284) gv1098: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc848: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1098, R.dtype("float16")) _847: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape285, alloc848) R.vm.kill_object(reshape285) gv1099: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape286: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc848, gv1099, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc848) gv1100: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape287: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape286, gv1100, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape286) gpt_neox_layers_23_post_attention_layernorm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[372] gpt_neox_layers_23_post_attention_layernorm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[373] gv1101: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc849: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1101, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc845, gpt_neox_layers_23_post_attention_layernorm_weight3, gpt_neox_layers_23_post_attention_layernorm_bias3, alloc849) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_weight3) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_bias3) gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight3: R.Tensor((4096, 128), dtype="uint32") = packed_params[380] gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale3: R.Tensor((4096, 32), dtype="float16") = packed_params[381] gpt_neox_layers_23_mlp_dense_h_to_4h_bias3: R.Tensor((4096,), dtype="float32") = packed_params[382] gv1102: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc850: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1102, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale3, alloc849, gpt_neox_layers_23_mlp_dense_h_to_4h_bias3, alloc850) R.vm.kill_object(alloc849) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight3) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale3) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_bias3) gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight3: R.Tensor((1024, 512), dtype="uint32") = packed_params[383] gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale3: R.Tensor((1024, 128), dtype="float16") = packed_params[384] gpt_neox_layers_23_mlp_dense_4h_to_h_bias3: R.Tensor((1024,), dtype="float32") = packed_params[385] gv1103: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc851: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1103, R.dtype("float16")) R.vm.kill_object(storage26) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale3, alloc850, gpt_neox_layers_23_mlp_dense_4h_to_h_bias3, alloc851) R.vm.kill_object(alloc850) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight3) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale3) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_bias3) gpt_neox_layers_23_attention_dense_q_weight3: R.Tensor((1024, 128), dtype="uint32") = packed_params[377] gpt_neox_layers_23_attention_dense_q_scale3: R.Tensor((1024, 32), dtype="float16") = packed_params[378] gpt_neox_layers_23_attention_dense_bias3: R.Tensor((1024,), dtype="float16") = packed_params[379] gv1104: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc852: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1104, R.dtype("float16")) R.vm.kill_object(storage28) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_23_attention_dense_q_weight3, gpt_neox_layers_23_attention_dense_q_scale3, reshape287, gpt_neox_layers_23_attention_dense_bias3, alloc851, alloc845, alloc852) R.vm.kill_object(alloc845) R.vm.kill_object(reshape287) R.vm.kill_object(alloc851) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_weight3) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_scale3) R.vm.kill_object(gpt_neox_layers_23_attention_dense_bias3) gpt_neox_final_layer_norm_weight3: R.Tensor((1024,), dtype="float16") = packed_params[386] gpt_neox_final_layer_norm_bias3: R.Tensor((1024,), dtype="float16") = packed_params[387] gv1105: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc853: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv1105, R.dtype("float16")) R.vm.kill_object(storage25) cls.gpt_neox_q4f16_1_layer_norm1(alloc852, gpt_neox_final_layer_norm_weight3, gpt_neox_final_layer_norm_bias3, alloc853) R.vm.kill_object(alloc852) R.vm.kill_object(gpt_neox_final_layer_norm_weight3) R.vm.kill_object(gpt_neox_final_layer_norm_bias3) storage29: R.Object = R.vm.alloc_storage(R.shape([262144]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv1106: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc854: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1106, R.dtype("float16")) R.vm.kill_object(storage29) cls.gpt_neox_q4f16_1_take(alloc853, logit_positions, alloc854) R.vm.kill_object(alloc853) embed_out_q_weight3: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] embed_out_q_scale3: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] gv1107: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=1),)) storage30: R.Object = R.vm.alloc_storage(gv1107, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv1108: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=3),)) alloc855: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage30, R.prim_value(0), gv1108, R.dtype("float32")) R.vm.kill_object(storage30) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5(embed_out_q_weight3, embed_out_q_scale3, alloc854, alloc855) R.vm.kill_object(alloc854) R.vm.kill_object(embed_out_q_weight3) R.vm.kill_object(embed_out_q_scale3) gv3: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc855, paged_kv_cache R.vm.kill_object(alloc855) gv1109: R.Tensor(dtype="float32", ndim=3) = gv3[0] R.call_packed("vm.builtin.match_shape", gv1109, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.prim_value(3), R.prim_value(2), R.str("ErrorContext(fn=batch_prefill, loc=return, annotation=R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv3 @R.function def batch_verify(input_embeds: R.Tensor((1, "seq_len", 1024), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(390), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv565: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv565, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv566: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv566, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv567: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] R.call_packed("vm.builtin.check_tensor_info", gv567, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv568: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] R.call_packed("vm.builtin.check_tensor_info", gv568, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), R.str("ErrorContext(fn=batch_verify, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv565, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv566, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv567, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv568, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func4(shape_heap) gpt_neox_layers_0_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[3] storage20: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv569: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc515: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv569, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_input_layernorm_weight5, gpt_neox_layers_0_input_layernorm_bias5, alloc515) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias5) gpt_neox_layers_0_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[8] storage21: R.Object = R.vm.alloc_storage(R.shape([12582912]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv570: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc516: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv570, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight5, gpt_neox_layers_0_attention_query_key_value_q_scale5, alloc515, gpt_neox_layers_0_attention_query_key_value_bias5, alloc516) R.vm.kill_object(alloc515) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias5) gv571: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape384: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc516, gv571, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc516) gv572: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape385: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape384, gv572, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape384) gv573: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc517: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv573, R.dtype("float16")) _516: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape385, alloc517) R.vm.kill_object(reshape385) gv574: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape386: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc517, gv574, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc517) gv575: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape387: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape386, gv575, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape386) gpt_neox_layers_0_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[5] gv576: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc518: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv576, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight5, gpt_neox_layers_0_post_attention_layernorm_bias5, alloc518) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias5) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[14] storage22: R.Object = R.vm.alloc_storage(R.shape([16777216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv577: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc519: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv577, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5, alloc518, gpt_neox_layers_0_mlp_dense_h_to_4h_bias5, alloc519) R.vm.kill_object(alloc518) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias5) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[17] gv578: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc520: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv578, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5, alloc519, gpt_neox_layers_0_mlp_dense_4h_to_h_bias5, alloc520) R.vm.kill_object(alloc519) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias5) gpt_neox_layers_0_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[11] gv579: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc521: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv579, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight5, gpt_neox_layers_0_attention_dense_q_scale5, reshape387, gpt_neox_layers_0_attention_dense_bias5, alloc520, input_embeds, alloc521) R.vm.kill_object(reshape387) R.vm.kill_object(alloc520) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias5) gpt_neox_layers_1_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[19] gv580: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc522: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv580, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc521, gpt_neox_layers_1_input_layernorm_weight5, gpt_neox_layers_1_input_layernorm_bias5, alloc522) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias5) gpt_neox_layers_1_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[24] gv581: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc523: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv581, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_1_attention_query_key_value_q_weight5, gpt_neox_layers_1_attention_query_key_value_q_scale5, alloc522, gpt_neox_layers_1_attention_query_key_value_bias5, alloc523) R.vm.kill_object(alloc522) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias5) gv582: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape388: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc523, gv582, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc523) gv583: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape389: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape388, gv583, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape388) gv584: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc524: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv584, R.dtype("float16")) _523: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape389, alloc524) R.vm.kill_object(reshape389) gv585: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape390: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc524, gv585, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc524) gv586: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape391: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape390, gv586, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape390) gpt_neox_layers_1_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[21] gv587: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc525: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv587, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc521, gpt_neox_layers_1_post_attention_layernorm_weight5, gpt_neox_layers_1_post_attention_layernorm_bias5, alloc525) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias5) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[30] storage23: R.Object = R.vm.alloc_storage(R.shape([16777216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv588: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc526: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv588, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5, alloc525, gpt_neox_layers_1_mlp_dense_h_to_4h_bias5, alloc526) R.vm.kill_object(alloc525) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias5) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[33] gv589: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc527: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv589, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5, alloc526, gpt_neox_layers_1_mlp_dense_4h_to_h_bias5, alloc527) R.vm.kill_object(alloc526) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias5) gpt_neox_layers_1_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[27] gv590: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc528: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv590, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_1_attention_dense_q_weight5, gpt_neox_layers_1_attention_dense_q_scale5, reshape391, gpt_neox_layers_1_attention_dense_bias5, alloc527, alloc521, alloc528) R.vm.kill_object(alloc521) R.vm.kill_object(reshape391) R.vm.kill_object(alloc527) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias5) gpt_neox_layers_2_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[35] gv591: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc529: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv591, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc528, gpt_neox_layers_2_input_layernorm_weight5, gpt_neox_layers_2_input_layernorm_bias5, alloc529) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias5) gpt_neox_layers_2_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[40] gv592: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc530: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv592, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_2_attention_query_key_value_q_weight5, gpt_neox_layers_2_attention_query_key_value_q_scale5, alloc529, gpt_neox_layers_2_attention_query_key_value_bias5, alloc530) R.vm.kill_object(alloc529) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias5) gv593: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape392: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc530, gv593, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc530) gv594: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape393: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape392, gv594, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape392) gv595: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc531: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv595, R.dtype("float16")) _530: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape393, alloc531) R.vm.kill_object(reshape393) gv596: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape394: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc531, gv596, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc531) gv597: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape395: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape394, gv597, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape394) gpt_neox_layers_2_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[37] gv598: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc532: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv598, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc528, gpt_neox_layers_2_post_attention_layernorm_weight5, gpt_neox_layers_2_post_attention_layernorm_bias5, alloc532) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias5) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[46] gv599: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc533: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv599, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5, alloc532, gpt_neox_layers_2_mlp_dense_h_to_4h_bias5, alloc533) R.vm.kill_object(alloc532) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias5) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[49] gv600: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc534: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv600, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5, alloc533, gpt_neox_layers_2_mlp_dense_4h_to_h_bias5, alloc534) R.vm.kill_object(alloc533) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias5) gpt_neox_layers_2_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[43] gv601: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc535: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv601, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_2_attention_dense_q_weight5, gpt_neox_layers_2_attention_dense_q_scale5, reshape395, gpt_neox_layers_2_attention_dense_bias5, alloc534, alloc528, alloc535) R.vm.kill_object(alloc528) R.vm.kill_object(reshape395) R.vm.kill_object(alloc534) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias5) gpt_neox_layers_3_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[51] gv602: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc536: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv602, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc535, gpt_neox_layers_3_input_layernorm_weight5, gpt_neox_layers_3_input_layernorm_bias5, alloc536) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias5) gpt_neox_layers_3_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[56] gv603: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc537: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv603, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_3_attention_query_key_value_q_weight5, gpt_neox_layers_3_attention_query_key_value_q_scale5, alloc536, gpt_neox_layers_3_attention_query_key_value_bias5, alloc537) R.vm.kill_object(alloc536) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias5) gv604: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape396: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc537, gv604, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc537) gv605: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape397: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape396, gv605, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape396) gv606: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc538: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv606, R.dtype("float16")) _537: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape397, alloc538) R.vm.kill_object(reshape397) gv607: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape398: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc538, gv607, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc538) gv608: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape399: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape398, gv608, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape398) gpt_neox_layers_3_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[53] gv609: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc539: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv609, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc535, gpt_neox_layers_3_post_attention_layernorm_weight5, gpt_neox_layers_3_post_attention_layernorm_bias5, alloc539) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias5) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[62] gv610: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc540: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv610, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5, alloc539, gpt_neox_layers_3_mlp_dense_h_to_4h_bias5, alloc540) R.vm.kill_object(alloc539) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias5) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[65] gv611: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc541: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv611, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5, alloc540, gpt_neox_layers_3_mlp_dense_4h_to_h_bias5, alloc541) R.vm.kill_object(alloc540) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias5) gpt_neox_layers_3_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[59] gv612: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc542: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv612, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_3_attention_dense_q_weight5, gpt_neox_layers_3_attention_dense_q_scale5, reshape399, gpt_neox_layers_3_attention_dense_bias5, alloc541, alloc535, alloc542) R.vm.kill_object(alloc535) R.vm.kill_object(reshape399) R.vm.kill_object(alloc541) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias5) gpt_neox_layers_4_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[67] gv613: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc543: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv613, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc542, gpt_neox_layers_4_input_layernorm_weight5, gpt_neox_layers_4_input_layernorm_bias5, alloc543) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias5) gpt_neox_layers_4_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[72] gv614: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc544: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv614, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_4_attention_query_key_value_q_weight5, gpt_neox_layers_4_attention_query_key_value_q_scale5, alloc543, gpt_neox_layers_4_attention_query_key_value_bias5, alloc544) R.vm.kill_object(alloc543) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias5) gv615: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape400: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc544, gv615, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc544) gv616: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape401: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape400, gv616, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape400) gv617: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc545: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv617, R.dtype("float16")) _544: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape401, alloc545) R.vm.kill_object(reshape401) gv618: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape402: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc545, gv618, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc545) gv619: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape403: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape402, gv619, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape402) gpt_neox_layers_4_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[69] gv620: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc546: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv620, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc542, gpt_neox_layers_4_post_attention_layernorm_weight5, gpt_neox_layers_4_post_attention_layernorm_bias5, alloc546) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias5) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[78] gv621: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc547: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv621, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5, alloc546, gpt_neox_layers_4_mlp_dense_h_to_4h_bias5, alloc547) R.vm.kill_object(alloc546) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias5) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[81] gv622: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc548: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv622, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5, alloc547, gpt_neox_layers_4_mlp_dense_4h_to_h_bias5, alloc548) R.vm.kill_object(alloc547) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias5) gpt_neox_layers_4_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[75] gv623: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc549: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv623, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_4_attention_dense_q_weight5, gpt_neox_layers_4_attention_dense_q_scale5, reshape403, gpt_neox_layers_4_attention_dense_bias5, alloc548, alloc542, alloc549) R.vm.kill_object(alloc542) R.vm.kill_object(reshape403) R.vm.kill_object(alloc548) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias5) gpt_neox_layers_5_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[83] gv624: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc550: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv624, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc549, gpt_neox_layers_5_input_layernorm_weight5, gpt_neox_layers_5_input_layernorm_bias5, alloc550) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias5) gpt_neox_layers_5_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[88] gv625: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc551: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv625, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_5_attention_query_key_value_q_weight5, gpt_neox_layers_5_attention_query_key_value_q_scale5, alloc550, gpt_neox_layers_5_attention_query_key_value_bias5, alloc551) R.vm.kill_object(alloc550) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias5) gv626: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape404: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc551, gv626, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc551) gv627: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape405: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape404, gv627, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape404) gv628: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc552: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv628, R.dtype("float16")) _551: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape405, alloc552) R.vm.kill_object(reshape405) gv629: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape406: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc552, gv629, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc552) gv630: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape407: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape406, gv630, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape406) gpt_neox_layers_5_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[85] gv631: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc553: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv631, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc549, gpt_neox_layers_5_post_attention_layernorm_weight5, gpt_neox_layers_5_post_attention_layernorm_bias5, alloc553) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias5) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[94] gv632: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc554: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv632, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5, alloc553, gpt_neox_layers_5_mlp_dense_h_to_4h_bias5, alloc554) R.vm.kill_object(alloc553) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias5) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[97] gv633: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc555: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv633, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5, alloc554, gpt_neox_layers_5_mlp_dense_4h_to_h_bias5, alloc555) R.vm.kill_object(alloc554) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias5) gpt_neox_layers_5_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[91] gv634: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc556: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv634, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_5_attention_dense_q_weight5, gpt_neox_layers_5_attention_dense_q_scale5, reshape407, gpt_neox_layers_5_attention_dense_bias5, alloc555, alloc549, alloc556) R.vm.kill_object(alloc549) R.vm.kill_object(reshape407) R.vm.kill_object(alloc555) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias5) gpt_neox_layers_6_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[98] gpt_neox_layers_6_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[99] gv635: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc557: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv635, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc556, gpt_neox_layers_6_input_layernorm_weight5, gpt_neox_layers_6_input_layernorm_bias5, alloc557) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_bias5) gpt_neox_layers_6_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[102] gpt_neox_layers_6_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[103] gpt_neox_layers_6_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[104] gv636: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc558: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv636, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_6_attention_query_key_value_q_weight5, gpt_neox_layers_6_attention_query_key_value_q_scale5, alloc557, gpt_neox_layers_6_attention_query_key_value_bias5, alloc558) R.vm.kill_object(alloc557) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_bias5) gv637: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape408: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc558, gv637, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc558) gv638: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape409: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape408, gv638, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape408) gv639: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc559: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv639, R.dtype("float16")) _558: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape409, alloc559) R.vm.kill_object(reshape409) gv640: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape410: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc559, gv640, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc559) gv641: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape411: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape410, gv641, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape410) gpt_neox_layers_6_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[100] gpt_neox_layers_6_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[101] gv642: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc560: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv642, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc556, gpt_neox_layers_6_post_attention_layernorm_weight5, gpt_neox_layers_6_post_attention_layernorm_bias5, alloc560) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_bias5) gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[108] gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[109] gpt_neox_layers_6_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[110] gv643: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc561: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv643, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale5, alloc560, gpt_neox_layers_6_mlp_dense_h_to_4h_bias5, alloc561) R.vm.kill_object(alloc560) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_bias5) gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[111] gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[112] gpt_neox_layers_6_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[113] gv644: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc562: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv644, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale5, alloc561, gpt_neox_layers_6_mlp_dense_4h_to_h_bias5, alloc562) R.vm.kill_object(alloc561) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_bias5) gpt_neox_layers_6_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[105] gpt_neox_layers_6_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[106] gpt_neox_layers_6_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[107] gv645: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc563: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv645, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_6_attention_dense_q_weight5, gpt_neox_layers_6_attention_dense_q_scale5, reshape411, gpt_neox_layers_6_attention_dense_bias5, alloc562, alloc556, alloc563) R.vm.kill_object(alloc556) R.vm.kill_object(reshape411) R.vm.kill_object(alloc562) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_6_attention_dense_bias5) gpt_neox_layers_7_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[114] gpt_neox_layers_7_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[115] gv646: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc564: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv646, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc563, gpt_neox_layers_7_input_layernorm_weight5, gpt_neox_layers_7_input_layernorm_bias5, alloc564) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_bias5) gpt_neox_layers_7_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[118] gpt_neox_layers_7_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[119] gpt_neox_layers_7_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[120] gv647: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc565: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv647, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_7_attention_query_key_value_q_weight5, gpt_neox_layers_7_attention_query_key_value_q_scale5, alloc564, gpt_neox_layers_7_attention_query_key_value_bias5, alloc565) R.vm.kill_object(alloc564) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_bias5) gv648: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape412: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc565, gv648, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc565) gv649: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape413: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape412, gv649, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape412) gv650: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc566: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv650, R.dtype("float16")) _565: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape413, alloc566) R.vm.kill_object(reshape413) gv651: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape414: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc566, gv651, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc566) gv652: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape415: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape414, gv652, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape414) gpt_neox_layers_7_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[116] gpt_neox_layers_7_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[117] gv653: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc567: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv653, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc563, gpt_neox_layers_7_post_attention_layernorm_weight5, gpt_neox_layers_7_post_attention_layernorm_bias5, alloc567) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_bias5) gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[124] gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[125] gpt_neox_layers_7_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[126] gv654: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc568: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv654, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale5, alloc567, gpt_neox_layers_7_mlp_dense_h_to_4h_bias5, alloc568) R.vm.kill_object(alloc567) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_bias5) gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[127] gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[128] gpt_neox_layers_7_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[129] gv655: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc569: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv655, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale5, alloc568, gpt_neox_layers_7_mlp_dense_4h_to_h_bias5, alloc569) R.vm.kill_object(alloc568) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_bias5) gpt_neox_layers_7_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[121] gpt_neox_layers_7_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[122] gpt_neox_layers_7_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[123] gv656: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc570: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv656, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_7_attention_dense_q_weight5, gpt_neox_layers_7_attention_dense_q_scale5, reshape415, gpt_neox_layers_7_attention_dense_bias5, alloc569, alloc563, alloc570) R.vm.kill_object(alloc563) R.vm.kill_object(reshape415) R.vm.kill_object(alloc569) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_7_attention_dense_bias5) gpt_neox_layers_8_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[130] gpt_neox_layers_8_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[131] gv657: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc571: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv657, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc570, gpt_neox_layers_8_input_layernorm_weight5, gpt_neox_layers_8_input_layernorm_bias5, alloc571) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_bias5) gpt_neox_layers_8_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[134] gpt_neox_layers_8_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[135] gpt_neox_layers_8_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[136] gv658: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc572: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv658, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_8_attention_query_key_value_q_weight5, gpt_neox_layers_8_attention_query_key_value_q_scale5, alloc571, gpt_neox_layers_8_attention_query_key_value_bias5, alloc572) R.vm.kill_object(alloc571) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_bias5) gv659: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape416: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc572, gv659, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc572) gv660: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape417: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape416, gv660, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape416) gv661: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc573: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv661, R.dtype("float16")) _572: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape417, alloc573) R.vm.kill_object(reshape417) gv662: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape418: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc573, gv662, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc573) gv663: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape419: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape418, gv663, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape418) gpt_neox_layers_8_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[132] gpt_neox_layers_8_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[133] gv664: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc574: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv664, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc570, gpt_neox_layers_8_post_attention_layernorm_weight5, gpt_neox_layers_8_post_attention_layernorm_bias5, alloc574) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_bias5) gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[140] gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[141] gpt_neox_layers_8_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[142] gv665: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc575: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv665, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale5, alloc574, gpt_neox_layers_8_mlp_dense_h_to_4h_bias5, alloc575) R.vm.kill_object(alloc574) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_bias5) gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[143] gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[144] gpt_neox_layers_8_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[145] gv666: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc576: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv666, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale5, alloc575, gpt_neox_layers_8_mlp_dense_4h_to_h_bias5, alloc576) R.vm.kill_object(alloc575) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_bias5) gpt_neox_layers_8_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[137] gpt_neox_layers_8_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[138] gpt_neox_layers_8_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[139] gv667: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc577: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv667, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_8_attention_dense_q_weight5, gpt_neox_layers_8_attention_dense_q_scale5, reshape419, gpt_neox_layers_8_attention_dense_bias5, alloc576, alloc570, alloc577) R.vm.kill_object(alloc570) R.vm.kill_object(reshape419) R.vm.kill_object(alloc576) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_8_attention_dense_bias5) gpt_neox_layers_9_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[146] gpt_neox_layers_9_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[147] gv668: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc578: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv668, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc577, gpt_neox_layers_9_input_layernorm_weight5, gpt_neox_layers_9_input_layernorm_bias5, alloc578) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_bias5) gpt_neox_layers_9_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[150] gpt_neox_layers_9_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[151] gpt_neox_layers_9_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[152] gv669: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc579: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv669, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_9_attention_query_key_value_q_weight5, gpt_neox_layers_9_attention_query_key_value_q_scale5, alloc578, gpt_neox_layers_9_attention_query_key_value_bias5, alloc579) R.vm.kill_object(alloc578) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_bias5) gv670: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape420: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc579, gv670, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc579) gv671: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape421: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape420, gv671, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape420) gv672: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc580: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv672, R.dtype("float16")) _579: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape421, alloc580) R.vm.kill_object(reshape421) gv673: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape422: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc580, gv673, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc580) gv674: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape423: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape422, gv674, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape422) gpt_neox_layers_9_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[148] gpt_neox_layers_9_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[149] gv675: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc581: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv675, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc577, gpt_neox_layers_9_post_attention_layernorm_weight5, gpt_neox_layers_9_post_attention_layernorm_bias5, alloc581) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_bias5) gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[156] gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[157] gpt_neox_layers_9_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[158] gv676: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc582: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv676, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale5, alloc581, gpt_neox_layers_9_mlp_dense_h_to_4h_bias5, alloc582) R.vm.kill_object(alloc581) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_bias5) gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[159] gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[160] gpt_neox_layers_9_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[161] gv677: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc583: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv677, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale5, alloc582, gpt_neox_layers_9_mlp_dense_4h_to_h_bias5, alloc583) R.vm.kill_object(alloc582) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_bias5) gpt_neox_layers_9_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[153] gpt_neox_layers_9_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[154] gpt_neox_layers_9_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[155] gv678: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc584: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv678, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_9_attention_dense_q_weight5, gpt_neox_layers_9_attention_dense_q_scale5, reshape423, gpt_neox_layers_9_attention_dense_bias5, alloc583, alloc577, alloc584) R.vm.kill_object(alloc577) R.vm.kill_object(reshape423) R.vm.kill_object(alloc583) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_9_attention_dense_bias5) gpt_neox_layers_10_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[162] gpt_neox_layers_10_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[163] gv679: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc585: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv679, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc584, gpt_neox_layers_10_input_layernorm_weight5, gpt_neox_layers_10_input_layernorm_bias5, alloc585) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_bias5) gpt_neox_layers_10_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[166] gpt_neox_layers_10_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[167] gpt_neox_layers_10_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[168] gv680: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc586: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv680, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_10_attention_query_key_value_q_weight5, gpt_neox_layers_10_attention_query_key_value_q_scale5, alloc585, gpt_neox_layers_10_attention_query_key_value_bias5, alloc586) R.vm.kill_object(alloc585) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_bias5) gv681: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape424: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc586, gv681, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc586) gv682: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape425: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape424, gv682, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape424) gv683: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc587: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv683, R.dtype("float16")) _586: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape425, alloc587) R.vm.kill_object(reshape425) gv684: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape426: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc587, gv684, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc587) gv685: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape427: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape426, gv685, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape426) gpt_neox_layers_10_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[164] gpt_neox_layers_10_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[165] gv686: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc588: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv686, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc584, gpt_neox_layers_10_post_attention_layernorm_weight5, gpt_neox_layers_10_post_attention_layernorm_bias5, alloc588) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_bias5) gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[172] gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[173] gpt_neox_layers_10_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[174] gv687: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc589: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv687, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale5, alloc588, gpt_neox_layers_10_mlp_dense_h_to_4h_bias5, alloc589) R.vm.kill_object(alloc588) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_bias5) gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[175] gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[176] gpt_neox_layers_10_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[177] gv688: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc590: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv688, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale5, alloc589, gpt_neox_layers_10_mlp_dense_4h_to_h_bias5, alloc590) R.vm.kill_object(alloc589) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_bias5) gpt_neox_layers_10_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[169] gpt_neox_layers_10_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[170] gpt_neox_layers_10_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[171] gv689: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc591: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv689, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_10_attention_dense_q_weight5, gpt_neox_layers_10_attention_dense_q_scale5, reshape427, gpt_neox_layers_10_attention_dense_bias5, alloc590, alloc584, alloc591) R.vm.kill_object(alloc584) R.vm.kill_object(reshape427) R.vm.kill_object(alloc590) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_10_attention_dense_bias5) gpt_neox_layers_11_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[178] gpt_neox_layers_11_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[179] gv690: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc592: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv690, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc591, gpt_neox_layers_11_input_layernorm_weight5, gpt_neox_layers_11_input_layernorm_bias5, alloc592) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_bias5) gpt_neox_layers_11_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[182] gpt_neox_layers_11_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[183] gpt_neox_layers_11_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[184] gv691: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc593: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv691, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_11_attention_query_key_value_q_weight5, gpt_neox_layers_11_attention_query_key_value_q_scale5, alloc592, gpt_neox_layers_11_attention_query_key_value_bias5, alloc593) R.vm.kill_object(alloc592) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_bias5) gv692: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape428: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc593, gv692, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc593) gv693: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape429: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape428, gv693, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape428) gv694: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc594: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv694, R.dtype("float16")) _593: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape429, alloc594) R.vm.kill_object(reshape429) gv695: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape430: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc594, gv695, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc594) gv696: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape431: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape430, gv696, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape430) gpt_neox_layers_11_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[180] gpt_neox_layers_11_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[181] gv697: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc595: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv697, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc591, gpt_neox_layers_11_post_attention_layernorm_weight5, gpt_neox_layers_11_post_attention_layernorm_bias5, alloc595) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_bias5) gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[188] gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[189] gpt_neox_layers_11_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[190] gv698: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc596: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv698, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale5, alloc595, gpt_neox_layers_11_mlp_dense_h_to_4h_bias5, alloc596) R.vm.kill_object(alloc595) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_bias5) gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[191] gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[192] gpt_neox_layers_11_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[193] gv699: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc597: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv699, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale5, alloc596, gpt_neox_layers_11_mlp_dense_4h_to_h_bias5, alloc597) R.vm.kill_object(alloc596) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_bias5) gpt_neox_layers_11_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[185] gpt_neox_layers_11_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[186] gpt_neox_layers_11_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[187] gv700: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc598: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv700, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_11_attention_dense_q_weight5, gpt_neox_layers_11_attention_dense_q_scale5, reshape431, gpt_neox_layers_11_attention_dense_bias5, alloc597, alloc591, alloc598) R.vm.kill_object(alloc591) R.vm.kill_object(reshape431) R.vm.kill_object(alloc597) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_11_attention_dense_bias5) gpt_neox_layers_12_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[194] gpt_neox_layers_12_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[195] gv701: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc599: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv701, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc598, gpt_neox_layers_12_input_layernorm_weight5, gpt_neox_layers_12_input_layernorm_bias5, alloc599) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_bias5) gpt_neox_layers_12_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[198] gpt_neox_layers_12_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[199] gpt_neox_layers_12_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[200] gv702: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc600: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv702, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_12_attention_query_key_value_q_weight5, gpt_neox_layers_12_attention_query_key_value_q_scale5, alloc599, gpt_neox_layers_12_attention_query_key_value_bias5, alloc600) R.vm.kill_object(alloc599) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_bias5) gv703: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape432: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc600, gv703, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc600) gv704: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape433: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape432, gv704, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape432) gv705: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc601: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv705, R.dtype("float16")) _600: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape433, alloc601) R.vm.kill_object(reshape433) gv706: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape434: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc601, gv706, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc601) gv707: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape435: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape434, gv707, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape434) gpt_neox_layers_12_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[196] gpt_neox_layers_12_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[197] gv708: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc602: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv708, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc598, gpt_neox_layers_12_post_attention_layernorm_weight5, gpt_neox_layers_12_post_attention_layernorm_bias5, alloc602) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_bias5) gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[204] gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[205] gpt_neox_layers_12_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[206] gv709: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc603: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv709, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale5, alloc602, gpt_neox_layers_12_mlp_dense_h_to_4h_bias5, alloc603) R.vm.kill_object(alloc602) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_bias5) gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[207] gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[208] gpt_neox_layers_12_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[209] gv710: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc604: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv710, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale5, alloc603, gpt_neox_layers_12_mlp_dense_4h_to_h_bias5, alloc604) R.vm.kill_object(alloc603) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_bias5) gpt_neox_layers_12_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[201] gpt_neox_layers_12_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[202] gpt_neox_layers_12_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[203] gv711: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc605: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv711, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_12_attention_dense_q_weight5, gpt_neox_layers_12_attention_dense_q_scale5, reshape435, gpt_neox_layers_12_attention_dense_bias5, alloc604, alloc598, alloc605) R.vm.kill_object(alloc598) R.vm.kill_object(reshape435) R.vm.kill_object(alloc604) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_12_attention_dense_bias5) gpt_neox_layers_13_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[210] gpt_neox_layers_13_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[211] gv712: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc606: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv712, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc605, gpt_neox_layers_13_input_layernorm_weight5, gpt_neox_layers_13_input_layernorm_bias5, alloc606) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_bias5) gpt_neox_layers_13_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[214] gpt_neox_layers_13_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[215] gpt_neox_layers_13_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[216] gv713: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc607: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv713, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_13_attention_query_key_value_q_weight5, gpt_neox_layers_13_attention_query_key_value_q_scale5, alloc606, gpt_neox_layers_13_attention_query_key_value_bias5, alloc607) R.vm.kill_object(alloc606) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_bias5) gv714: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape436: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc607, gv714, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc607) gv715: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape437: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape436, gv715, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape436) gv716: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc608: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv716, R.dtype("float16")) _607: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape437, alloc608) R.vm.kill_object(reshape437) gv717: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape438: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc608, gv717, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc608) gv718: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape439: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape438, gv718, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape438) gpt_neox_layers_13_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[212] gpt_neox_layers_13_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[213] gv719: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc609: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv719, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc605, gpt_neox_layers_13_post_attention_layernorm_weight5, gpt_neox_layers_13_post_attention_layernorm_bias5, alloc609) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_bias5) gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[220] gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[221] gpt_neox_layers_13_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[222] gv720: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc610: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv720, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale5, alloc609, gpt_neox_layers_13_mlp_dense_h_to_4h_bias5, alloc610) R.vm.kill_object(alloc609) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_bias5) gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[223] gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[224] gpt_neox_layers_13_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[225] gv721: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc611: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv721, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale5, alloc610, gpt_neox_layers_13_mlp_dense_4h_to_h_bias5, alloc611) R.vm.kill_object(alloc610) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_bias5) gpt_neox_layers_13_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[217] gpt_neox_layers_13_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[218] gpt_neox_layers_13_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[219] gv722: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc612: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv722, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_13_attention_dense_q_weight5, gpt_neox_layers_13_attention_dense_q_scale5, reshape439, gpt_neox_layers_13_attention_dense_bias5, alloc611, alloc605, alloc612) R.vm.kill_object(alloc605) R.vm.kill_object(reshape439) R.vm.kill_object(alloc611) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_13_attention_dense_bias5) gpt_neox_layers_14_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[226] gpt_neox_layers_14_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[227] gv723: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc613: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv723, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc612, gpt_neox_layers_14_input_layernorm_weight5, gpt_neox_layers_14_input_layernorm_bias5, alloc613) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_bias5) gpt_neox_layers_14_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[230] gpt_neox_layers_14_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[231] gpt_neox_layers_14_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[232] gv724: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc614: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv724, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_14_attention_query_key_value_q_weight5, gpt_neox_layers_14_attention_query_key_value_q_scale5, alloc613, gpt_neox_layers_14_attention_query_key_value_bias5, alloc614) R.vm.kill_object(alloc613) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_bias5) gv725: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape440: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc614, gv725, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc614) gv726: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape441: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape440, gv726, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape440) gv727: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc615: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv727, R.dtype("float16")) _614: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape441, alloc615) R.vm.kill_object(reshape441) gv728: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape442: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc615, gv728, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc615) gv729: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape443: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape442, gv729, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape442) gpt_neox_layers_14_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[228] gpt_neox_layers_14_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[229] gv730: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc616: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv730, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc612, gpt_neox_layers_14_post_attention_layernorm_weight5, gpt_neox_layers_14_post_attention_layernorm_bias5, alloc616) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_bias5) gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[236] gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[237] gpt_neox_layers_14_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[238] gv731: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc617: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv731, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale5, alloc616, gpt_neox_layers_14_mlp_dense_h_to_4h_bias5, alloc617) R.vm.kill_object(alloc616) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_bias5) gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[239] gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[240] gpt_neox_layers_14_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[241] gv732: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc618: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv732, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale5, alloc617, gpt_neox_layers_14_mlp_dense_4h_to_h_bias5, alloc618) R.vm.kill_object(alloc617) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_bias5) gpt_neox_layers_14_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[233] gpt_neox_layers_14_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[234] gpt_neox_layers_14_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[235] gv733: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc619: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv733, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_14_attention_dense_q_weight5, gpt_neox_layers_14_attention_dense_q_scale5, reshape443, gpt_neox_layers_14_attention_dense_bias5, alloc618, alloc612, alloc619) R.vm.kill_object(alloc612) R.vm.kill_object(reshape443) R.vm.kill_object(alloc618) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_14_attention_dense_bias5) gpt_neox_layers_15_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[242] gpt_neox_layers_15_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[243] gv734: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc620: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv734, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc619, gpt_neox_layers_15_input_layernorm_weight5, gpt_neox_layers_15_input_layernorm_bias5, alloc620) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_bias5) gpt_neox_layers_15_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[246] gpt_neox_layers_15_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[247] gpt_neox_layers_15_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[248] gv735: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc621: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv735, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_15_attention_query_key_value_q_weight5, gpt_neox_layers_15_attention_query_key_value_q_scale5, alloc620, gpt_neox_layers_15_attention_query_key_value_bias5, alloc621) R.vm.kill_object(alloc620) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_bias5) gv736: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape444: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc621, gv736, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc621) gv737: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape445: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape444, gv737, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape444) gv738: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc622: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv738, R.dtype("float16")) _621: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape445, alloc622) R.vm.kill_object(reshape445) gv739: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape446: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc622, gv739, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc622) gv740: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape447: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape446, gv740, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape446) gpt_neox_layers_15_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[244] gpt_neox_layers_15_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[245] gv741: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc623: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv741, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc619, gpt_neox_layers_15_post_attention_layernorm_weight5, gpt_neox_layers_15_post_attention_layernorm_bias5, alloc623) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_bias5) gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[252] gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[253] gpt_neox_layers_15_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[254] gv742: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc624: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv742, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale5, alloc623, gpt_neox_layers_15_mlp_dense_h_to_4h_bias5, alloc624) R.vm.kill_object(alloc623) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_bias5) gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[255] gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[256] gpt_neox_layers_15_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[257] gv743: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc625: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv743, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale5, alloc624, gpt_neox_layers_15_mlp_dense_4h_to_h_bias5, alloc625) R.vm.kill_object(alloc624) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_bias5) gpt_neox_layers_15_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[249] gpt_neox_layers_15_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[250] gpt_neox_layers_15_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[251] gv744: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc626: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv744, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_15_attention_dense_q_weight5, gpt_neox_layers_15_attention_dense_q_scale5, reshape447, gpt_neox_layers_15_attention_dense_bias5, alloc625, alloc619, alloc626) R.vm.kill_object(alloc619) R.vm.kill_object(reshape447) R.vm.kill_object(alloc625) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_15_attention_dense_bias5) gpt_neox_layers_16_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[258] gpt_neox_layers_16_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[259] gv745: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc627: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv745, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc626, gpt_neox_layers_16_input_layernorm_weight5, gpt_neox_layers_16_input_layernorm_bias5, alloc627) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_bias5) gpt_neox_layers_16_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[262] gpt_neox_layers_16_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[263] gpt_neox_layers_16_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[264] gv746: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc628: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv746, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_16_attention_query_key_value_q_weight5, gpt_neox_layers_16_attention_query_key_value_q_scale5, alloc627, gpt_neox_layers_16_attention_query_key_value_bias5, alloc628) R.vm.kill_object(alloc627) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_bias5) gv747: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape448: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc628, gv747, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc628) gv748: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape449: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape448, gv748, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape448) gv749: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc629: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv749, R.dtype("float16")) _628: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape449, alloc629) R.vm.kill_object(reshape449) gv750: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape450: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc629, gv750, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc629) gv751: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape451: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape450, gv751, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape450) gpt_neox_layers_16_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[260] gpt_neox_layers_16_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[261] gv752: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc630: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv752, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc626, gpt_neox_layers_16_post_attention_layernorm_weight5, gpt_neox_layers_16_post_attention_layernorm_bias5, alloc630) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_bias5) gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[268] gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[269] gpt_neox_layers_16_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[270] gv753: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc631: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv753, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale5, alloc630, gpt_neox_layers_16_mlp_dense_h_to_4h_bias5, alloc631) R.vm.kill_object(alloc630) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_bias5) gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[271] gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[272] gpt_neox_layers_16_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[273] gv754: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc632: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv754, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale5, alloc631, gpt_neox_layers_16_mlp_dense_4h_to_h_bias5, alloc632) R.vm.kill_object(alloc631) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_bias5) gpt_neox_layers_16_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[265] gpt_neox_layers_16_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[266] gpt_neox_layers_16_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[267] gv755: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc633: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv755, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_16_attention_dense_q_weight5, gpt_neox_layers_16_attention_dense_q_scale5, reshape451, gpt_neox_layers_16_attention_dense_bias5, alloc632, alloc626, alloc633) R.vm.kill_object(alloc626) R.vm.kill_object(reshape451) R.vm.kill_object(alloc632) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_16_attention_dense_bias5) gpt_neox_layers_17_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[274] gpt_neox_layers_17_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[275] gv756: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc634: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv756, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc633, gpt_neox_layers_17_input_layernorm_weight5, gpt_neox_layers_17_input_layernorm_bias5, alloc634) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_bias5) gpt_neox_layers_17_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[278] gpt_neox_layers_17_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[279] gpt_neox_layers_17_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[280] gv757: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc635: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv757, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_17_attention_query_key_value_q_weight5, gpt_neox_layers_17_attention_query_key_value_q_scale5, alloc634, gpt_neox_layers_17_attention_query_key_value_bias5, alloc635) R.vm.kill_object(alloc634) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_bias5) gv758: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape452: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc635, gv758, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc635) gv759: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape453: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape452, gv759, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape452) gv760: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc636: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv760, R.dtype("float16")) _635: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape453, alloc636) R.vm.kill_object(reshape453) gv761: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape454: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc636, gv761, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc636) gv762: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape455: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape454, gv762, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape454) gpt_neox_layers_17_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[276] gpt_neox_layers_17_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[277] gv763: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc637: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv763, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc633, gpt_neox_layers_17_post_attention_layernorm_weight5, gpt_neox_layers_17_post_attention_layernorm_bias5, alloc637) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_bias5) gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[284] gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[285] gpt_neox_layers_17_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[286] gv764: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc638: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv764, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale5, alloc637, gpt_neox_layers_17_mlp_dense_h_to_4h_bias5, alloc638) R.vm.kill_object(alloc637) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_bias5) gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[287] gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[288] gpt_neox_layers_17_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[289] gv765: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc639: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv765, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale5, alloc638, gpt_neox_layers_17_mlp_dense_4h_to_h_bias5, alloc639) R.vm.kill_object(alloc638) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_bias5) gpt_neox_layers_17_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[281] gpt_neox_layers_17_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[282] gpt_neox_layers_17_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[283] gv766: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc640: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv766, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_17_attention_dense_q_weight5, gpt_neox_layers_17_attention_dense_q_scale5, reshape455, gpt_neox_layers_17_attention_dense_bias5, alloc639, alloc633, alloc640) R.vm.kill_object(alloc633) R.vm.kill_object(reshape455) R.vm.kill_object(alloc639) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_17_attention_dense_bias5) gpt_neox_layers_18_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[290] gpt_neox_layers_18_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[291] gv767: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc641: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv767, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc640, gpt_neox_layers_18_input_layernorm_weight5, gpt_neox_layers_18_input_layernorm_bias5, alloc641) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_bias5) gpt_neox_layers_18_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[294] gpt_neox_layers_18_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[295] gpt_neox_layers_18_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[296] gv768: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc642: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv768, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_18_attention_query_key_value_q_weight5, gpt_neox_layers_18_attention_query_key_value_q_scale5, alloc641, gpt_neox_layers_18_attention_query_key_value_bias5, alloc642) R.vm.kill_object(alloc641) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_bias5) gv769: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape456: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc642, gv769, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc642) gv770: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape457: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape456, gv770, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape456) gv771: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc643: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv771, R.dtype("float16")) _642: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape457, alloc643) R.vm.kill_object(reshape457) gv772: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape458: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc643, gv772, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc643) gv773: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape459: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape458, gv773, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape458) gpt_neox_layers_18_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[292] gpt_neox_layers_18_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[293] gv774: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc644: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv774, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc640, gpt_neox_layers_18_post_attention_layernorm_weight5, gpt_neox_layers_18_post_attention_layernorm_bias5, alloc644) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_bias5) gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[300] gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[301] gpt_neox_layers_18_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[302] gv775: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc645: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv775, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale5, alloc644, gpt_neox_layers_18_mlp_dense_h_to_4h_bias5, alloc645) R.vm.kill_object(alloc644) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_bias5) gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[303] gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[304] gpt_neox_layers_18_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[305] gv776: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc646: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv776, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale5, alloc645, gpt_neox_layers_18_mlp_dense_4h_to_h_bias5, alloc646) R.vm.kill_object(alloc645) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_bias5) gpt_neox_layers_18_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[297] gpt_neox_layers_18_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[298] gpt_neox_layers_18_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[299] gv777: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc647: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv777, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_18_attention_dense_q_weight5, gpt_neox_layers_18_attention_dense_q_scale5, reshape459, gpt_neox_layers_18_attention_dense_bias5, alloc646, alloc640, alloc647) R.vm.kill_object(alloc640) R.vm.kill_object(reshape459) R.vm.kill_object(alloc646) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_18_attention_dense_bias5) gpt_neox_layers_19_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[306] gpt_neox_layers_19_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[307] gv778: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc648: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv778, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc647, gpt_neox_layers_19_input_layernorm_weight5, gpt_neox_layers_19_input_layernorm_bias5, alloc648) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_bias5) gpt_neox_layers_19_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[310] gpt_neox_layers_19_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[311] gpt_neox_layers_19_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[312] gv779: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc649: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv779, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_19_attention_query_key_value_q_weight5, gpt_neox_layers_19_attention_query_key_value_q_scale5, alloc648, gpt_neox_layers_19_attention_query_key_value_bias5, alloc649) R.vm.kill_object(alloc648) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_bias5) gv780: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape460: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc649, gv780, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc649) gv781: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape461: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape460, gv781, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape460) gv782: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc650: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv782, R.dtype("float16")) _649: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape461, alloc650) R.vm.kill_object(reshape461) gv783: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape462: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc650, gv783, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc650) gv784: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape463: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape462, gv784, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape462) gpt_neox_layers_19_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[308] gpt_neox_layers_19_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[309] gv785: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc651: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv785, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc647, gpt_neox_layers_19_post_attention_layernorm_weight5, gpt_neox_layers_19_post_attention_layernorm_bias5, alloc651) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_bias5) gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[316] gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[317] gpt_neox_layers_19_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[318] gv786: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc652: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv786, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale5, alloc651, gpt_neox_layers_19_mlp_dense_h_to_4h_bias5, alloc652) R.vm.kill_object(alloc651) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_bias5) gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[319] gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[320] gpt_neox_layers_19_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[321] gv787: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc653: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv787, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale5, alloc652, gpt_neox_layers_19_mlp_dense_4h_to_h_bias5, alloc653) R.vm.kill_object(alloc652) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_bias5) gpt_neox_layers_19_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[313] gpt_neox_layers_19_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[314] gpt_neox_layers_19_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[315] gv788: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc654: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv788, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_19_attention_dense_q_weight5, gpt_neox_layers_19_attention_dense_q_scale5, reshape463, gpt_neox_layers_19_attention_dense_bias5, alloc653, alloc647, alloc654) R.vm.kill_object(alloc647) R.vm.kill_object(reshape463) R.vm.kill_object(alloc653) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_19_attention_dense_bias5) gpt_neox_layers_20_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[322] gpt_neox_layers_20_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[323] gv789: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc655: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv789, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc654, gpt_neox_layers_20_input_layernorm_weight5, gpt_neox_layers_20_input_layernorm_bias5, alloc655) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_bias5) gpt_neox_layers_20_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[326] gpt_neox_layers_20_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[327] gpt_neox_layers_20_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[328] gv790: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc656: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv790, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_20_attention_query_key_value_q_weight5, gpt_neox_layers_20_attention_query_key_value_q_scale5, alloc655, gpt_neox_layers_20_attention_query_key_value_bias5, alloc656) R.vm.kill_object(alloc655) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_bias5) gv791: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape464: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc656, gv791, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc656) gv792: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape465: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape464, gv792, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape464) gv793: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc657: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv793, R.dtype("float16")) _656: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape465, alloc657) R.vm.kill_object(reshape465) gv794: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape466: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc657, gv794, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc657) gv795: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape467: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape466, gv795, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape466) gpt_neox_layers_20_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[324] gpt_neox_layers_20_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[325] gv796: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc658: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv796, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc654, gpt_neox_layers_20_post_attention_layernorm_weight5, gpt_neox_layers_20_post_attention_layernorm_bias5, alloc658) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_bias5) gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[332] gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[333] gpt_neox_layers_20_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[334] gv797: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc659: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv797, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale5, alloc658, gpt_neox_layers_20_mlp_dense_h_to_4h_bias5, alloc659) R.vm.kill_object(alloc658) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_bias5) gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[335] gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[336] gpt_neox_layers_20_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[337] gv798: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc660: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv798, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale5, alloc659, gpt_neox_layers_20_mlp_dense_4h_to_h_bias5, alloc660) R.vm.kill_object(alloc659) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_bias5) gpt_neox_layers_20_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[329] gpt_neox_layers_20_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[330] gpt_neox_layers_20_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[331] gv799: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc661: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv799, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_20_attention_dense_q_weight5, gpt_neox_layers_20_attention_dense_q_scale5, reshape467, gpt_neox_layers_20_attention_dense_bias5, alloc660, alloc654, alloc661) R.vm.kill_object(alloc654) R.vm.kill_object(reshape467) R.vm.kill_object(alloc660) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_20_attention_dense_bias5) gpt_neox_layers_21_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[338] gpt_neox_layers_21_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[339] gv800: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc662: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv800, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc661, gpt_neox_layers_21_input_layernorm_weight5, gpt_neox_layers_21_input_layernorm_bias5, alloc662) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_bias5) gpt_neox_layers_21_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[342] gpt_neox_layers_21_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[343] gpt_neox_layers_21_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[344] gv801: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc663: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv801, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_21_attention_query_key_value_q_weight5, gpt_neox_layers_21_attention_query_key_value_q_scale5, alloc662, gpt_neox_layers_21_attention_query_key_value_bias5, alloc663) R.vm.kill_object(alloc662) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_bias5) gv802: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape468: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc663, gv802, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc663) gv803: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape469: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape468, gv803, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape468) gv804: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc664: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv804, R.dtype("float16")) _663: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape469, alloc664) R.vm.kill_object(reshape469) gv805: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape470: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc664, gv805, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc664) gv806: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape471: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape470, gv806, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape470) gpt_neox_layers_21_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[340] gpt_neox_layers_21_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[341] gv807: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc665: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv807, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc661, gpt_neox_layers_21_post_attention_layernorm_weight5, gpt_neox_layers_21_post_attention_layernorm_bias5, alloc665) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_bias5) gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[348] gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[349] gpt_neox_layers_21_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[350] gv808: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc666: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv808, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale5, alloc665, gpt_neox_layers_21_mlp_dense_h_to_4h_bias5, alloc666) R.vm.kill_object(alloc665) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_bias5) gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[351] gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[352] gpt_neox_layers_21_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[353] gv809: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc667: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv809, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale5, alloc666, gpt_neox_layers_21_mlp_dense_4h_to_h_bias5, alloc667) R.vm.kill_object(alloc666) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_bias5) gpt_neox_layers_21_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[345] gpt_neox_layers_21_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[346] gpt_neox_layers_21_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[347] gv810: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc668: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv810, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_21_attention_dense_q_weight5, gpt_neox_layers_21_attention_dense_q_scale5, reshape471, gpt_neox_layers_21_attention_dense_bias5, alloc667, alloc661, alloc668) R.vm.kill_object(alloc661) R.vm.kill_object(reshape471) R.vm.kill_object(alloc667) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_21_attention_dense_bias5) gpt_neox_layers_22_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[354] gpt_neox_layers_22_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[355] gv811: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc669: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv811, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc668, gpt_neox_layers_22_input_layernorm_weight5, gpt_neox_layers_22_input_layernorm_bias5, alloc669) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_bias5) gpt_neox_layers_22_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[358] gpt_neox_layers_22_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[359] gpt_neox_layers_22_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[360] gv812: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc670: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv812, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_22_attention_query_key_value_q_weight5, gpt_neox_layers_22_attention_query_key_value_q_scale5, alloc669, gpt_neox_layers_22_attention_query_key_value_bias5, alloc670) R.vm.kill_object(alloc669) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_bias5) gv813: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape472: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc670, gv813, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc670) gv814: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape473: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape472, gv814, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape472) gv815: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc671: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv815, R.dtype("float16")) _670: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape473, alloc671) R.vm.kill_object(reshape473) gv816: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape474: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc671, gv816, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc671) gv817: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape475: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape474, gv817, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape474) gpt_neox_layers_22_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[356] gpt_neox_layers_22_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[357] gv818: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc672: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv818, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc668, gpt_neox_layers_22_post_attention_layernorm_weight5, gpt_neox_layers_22_post_attention_layernorm_bias5, alloc672) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_bias5) gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[364] gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[365] gpt_neox_layers_22_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[366] gv819: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc673: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv819, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale5, alloc672, gpt_neox_layers_22_mlp_dense_h_to_4h_bias5, alloc673) R.vm.kill_object(alloc672) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_bias5) gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[367] gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[368] gpt_neox_layers_22_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[369] gv820: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc674: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv820, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale5, alloc673, gpt_neox_layers_22_mlp_dense_4h_to_h_bias5, alloc674) R.vm.kill_object(alloc673) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_bias5) gpt_neox_layers_22_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[361] gpt_neox_layers_22_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[362] gpt_neox_layers_22_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[363] gv821: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc675: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage22, R.prim_value(0), gv821, R.dtype("float16")) R.vm.kill_object(storage22) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_22_attention_dense_q_weight5, gpt_neox_layers_22_attention_dense_q_scale5, reshape475, gpt_neox_layers_22_attention_dense_bias5, alloc674, alloc668, alloc675) R.vm.kill_object(alloc668) R.vm.kill_object(reshape475) R.vm.kill_object(alloc674) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_22_attention_dense_bias5) gpt_neox_layers_23_input_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[370] gpt_neox_layers_23_input_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[371] gv822: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc676: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv822, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc675, gpt_neox_layers_23_input_layernorm_weight5, gpt_neox_layers_23_input_layernorm_bias5, alloc676) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_bias5) gpt_neox_layers_23_attention_query_key_value_q_weight5: R.Tensor((3072, 128), dtype="uint32") = packed_params[374] gpt_neox_layers_23_attention_query_key_value_q_scale5: R.Tensor((3072, 32), dtype="float16") = packed_params[375] gpt_neox_layers_23_attention_query_key_value_bias5: R.Tensor((3072,), dtype="float16") = packed_params[376] gv823: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc677: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv823, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_23_attention_query_key_value_q_weight5, gpt_neox_layers_23_attention_query_key_value_q_scale5, alloc676, gpt_neox_layers_23_attention_query_key_value_bias5, alloc677) R.vm.kill_object(alloc676) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_weight5) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_scale5) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_bias5) gv824: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape476: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc677, gv824, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc677) gv825: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape477: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape476, gv825, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape476) gv826: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc678: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv826, R.dtype("float16")) _677: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape477, alloc678) R.vm.kill_object(reshape477) gv827: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape478: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc678, gv827, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc678) gv828: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape479: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape478, gv828, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape478) gpt_neox_layers_23_post_attention_layernorm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[372] gpt_neox_layers_23_post_attention_layernorm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[373] gv829: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc679: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv829, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc675, gpt_neox_layers_23_post_attention_layernorm_weight5, gpt_neox_layers_23_post_attention_layernorm_bias5, alloc679) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_weight5) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_bias5) gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight5: R.Tensor((4096, 128), dtype="uint32") = packed_params[380] gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale5: R.Tensor((4096, 32), dtype="float16") = packed_params[381] gpt_neox_layers_23_mlp_dense_h_to_4h_bias5: R.Tensor((4096,), dtype="float32") = packed_params[382] gv830: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc680: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv830, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale5, alloc679, gpt_neox_layers_23_mlp_dense_h_to_4h_bias5, alloc680) R.vm.kill_object(alloc679) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight5) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale5) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_bias5) gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight5: R.Tensor((1024, 512), dtype="uint32") = packed_params[383] gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale5: R.Tensor((1024, 128), dtype="float16") = packed_params[384] gpt_neox_layers_23_mlp_dense_4h_to_h_bias5: R.Tensor((1024,), dtype="float32") = packed_params[385] gv831: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc681: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage21, R.prim_value(0), gv831, R.dtype("float16")) R.vm.kill_object(storage21) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale5, alloc680, gpt_neox_layers_23_mlp_dense_4h_to_h_bias5, alloc681) R.vm.kill_object(alloc680) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight5) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale5) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_bias5) gpt_neox_layers_23_attention_dense_q_weight5: R.Tensor((1024, 128), dtype="uint32") = packed_params[377] gpt_neox_layers_23_attention_dense_q_scale5: R.Tensor((1024, 32), dtype="float16") = packed_params[378] gpt_neox_layers_23_attention_dense_bias5: R.Tensor((1024,), dtype="float16") = packed_params[379] gv832: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc682: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage23, R.prim_value(0), gv832, R.dtype("float16")) R.vm.kill_object(storage23) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_23_attention_dense_q_weight5, gpt_neox_layers_23_attention_dense_q_scale5, reshape479, gpt_neox_layers_23_attention_dense_bias5, alloc681, alloc675, alloc682) R.vm.kill_object(alloc675) R.vm.kill_object(reshape479) R.vm.kill_object(alloc681) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_weight5) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_scale5) R.vm.kill_object(gpt_neox_layers_23_attention_dense_bias5) gpt_neox_final_layer_norm_weight5: R.Tensor((1024,), dtype="float16") = packed_params[386] gpt_neox_final_layer_norm_bias5: R.Tensor((1024,), dtype="float16") = packed_params[387] gv833: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc683: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv833, R.dtype("float16")) R.vm.kill_object(storage20) cls.gpt_neox_q4f16_1_layer_norm1(alloc682, gpt_neox_final_layer_norm_weight5, gpt_neox_final_layer_norm_bias5, alloc683) R.vm.kill_object(alloc682) R.vm.kill_object(gpt_neox_final_layer_norm_weight5) R.vm.kill_object(gpt_neox_final_layer_norm_bias5) embed_out_q_weight5: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] embed_out_q_scale5: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] gv834: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage24: R.Object = R.vm.alloc_storage(gv834, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv835: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc684: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage24, R.prim_value(0), gv835, R.dtype("float32")) R.vm.kill_object(storage24) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5(embed_out_q_weight5, embed_out_q_scale5, alloc683, alloc684) R.vm.kill_object(alloc683) R.vm.kill_object(embed_out_q_weight5) R.vm.kill_object(embed_out_q_scale5) gv5: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc684, paged_kv_cache R.vm.kill_object(alloc684) gv836: R.Tensor(dtype="float32", ndim=3) = gv5[0] R.call_packed("vm.builtin.match_shape", gv836, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(0), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=batch_verify, loc=return, annotation=R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv5 @R.function def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: max_batch_size = T.int64() max_total_seq_len = T.int64() prefill_chunk_size = T.int64() page_size = T.int64() support_sliding_window = T.int64() R.func_attr({"relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_shape_info", max_batch_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[0], param=max_batch_size_, annotation=R.Shape([max_batch_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", max_total_seq_len_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[1], param=max_total_seq_len_, annotation=R.Shape([max_total_seq_len])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", prefill_chunk_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[2], param=prefill_chunk_size_, annotation=R.Shape([prefill_chunk_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", page_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[3], param=page_size_, annotation=R.Shape([page_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", support_sliding_window_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[4], param=support_sliding_window_, annotation=R.Shape([support_sliding_window])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", max_batch_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(0), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[0], param=max_batch_size_, annotation=R.Shape([max_batch_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", max_total_seq_len_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[1], param=max_total_seq_len_, annotation=R.Shape([max_total_seq_len])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", prefill_chunk_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[2], param=prefill_chunk_size_, annotation=R.Shape([prefill_chunk_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", page_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[3], param=page_size_, annotation=R.Shape([page_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", support_sliding_window_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[4], param=support_sliding_window_, annotation=R.Shape([support_sliding_window])) "), sinfo_args=(R.Tuple,)) gv556: R.Shape(ndim=5) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(5), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(2), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=5),)) paged_kv_cache: R.Object = R.call_packed("vm.builtin.paged_attention_kv_cache_create_reduced", gv556, R.shape([0, 24]), R.prim_value(16), R.prim_value(16), R.prim_value(64), R.prim_value(1), R.prim_value(1), R.prim_value(10000), R.const(0.0, "float16"), cls.gpt_neox_q4f16_1_tir_kv_cache_transpose_append, cls.gpt_neox_q4f16_1_batch_prefill_paged_kv, cls.gpt_neox_q4f16_1_batch_decode_paged_kv, cls.gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window, cls.gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window, cls.gpt_neox_q4f16_1_batch_prefill_ragged_kv, cls.gpt_neox_q4f16_1_merge_state_inplace, cls.gpt_neox_q4f16_1_fused_rope, cls.gpt_neox_q4f16_1_copy_single_page, cls.gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv, cls.gpt_neox_q4f16_1_compact_kv_copy, cls.gpt_neox_q4f16_1_batch_tree_attn, cls.gpt_neox_q4f16_1_tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) return paged_kv_cache @R.function def decode(input_embed: R.Tensor((1, 1, 1024), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(3),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embed, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[0], param=input_embed, annotation=R.Tensor((1, 1, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(390), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv549: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv549, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv550: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv550, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv551: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] R.call_packed("vm.builtin.check_tensor_info", gv551, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv552: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] R.call_packed("vm.builtin.check_tensor_info", gv552, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embed, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1024), R.str("ErrorContext(fn=decode, loc=param[0], param=input_embed, annotation=R.Tensor((1, 1, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv549, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv550, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv551, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv552, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func2(shape_heap) gpt_neox_layers_0_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[3] storage12: R.Object = R.vm.alloc_storage(R.shape([2048]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc342: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(input_embed, gpt_neox_layers_0_input_layernorm_weight2, gpt_neox_layers_0_input_layernorm_bias2, alloc342) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias2) gpt_neox_layers_0_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[8] storage13: R.Object = R.vm.alloc_storage(R.shape([6144]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc343: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_0_attention_query_key_value_q_weight2, gpt_neox_layers_0_attention_query_key_value_q_scale2, alloc342, gpt_neox_layers_0_attention_query_key_value_bias2, alloc343) R.vm.kill_object(alloc342) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias2) lv292: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc343, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc343) alloc344: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _344: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), lv292, alloc344) R.vm.kill_object(lv292) lv293: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc344, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc344) gpt_neox_layers_0_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[5] alloc345: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight2, gpt_neox_layers_0_post_attention_layernorm_bias2, alloc345) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias2) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[14] storage14: R.Object = R.vm.alloc_storage(R.shape([8192]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc346: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, alloc345, gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, alloc346) R.vm.kill_object(alloc345) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias2) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[17] alloc347: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, alloc346, gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, alloc347) R.vm.kill_object(alloc346) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias2) gpt_neox_layers_0_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[11] alloc348: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_0_attention_dense_q_weight2, gpt_neox_layers_0_attention_dense_q_scale2, lv293, gpt_neox_layers_0_attention_dense_bias2, alloc347, input_embed, alloc348) R.vm.kill_object(lv293) R.vm.kill_object(alloc347) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias2) gpt_neox_layers_1_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[19] alloc349: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc348, gpt_neox_layers_1_input_layernorm_weight2, gpt_neox_layers_1_input_layernorm_bias2, alloc349) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias2) gpt_neox_layers_1_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[24] alloc350: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_1_attention_query_key_value_q_weight2, gpt_neox_layers_1_attention_query_key_value_q_scale2, alloc349, gpt_neox_layers_1_attention_query_key_value_bias2, alloc350) R.vm.kill_object(alloc349) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias2) lv298: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc350, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc350) alloc351: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _351: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), lv298, alloc351) R.vm.kill_object(lv298) lv299: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc351, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc351) gpt_neox_layers_1_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[21] alloc352: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc348, gpt_neox_layers_1_post_attention_layernorm_weight2, gpt_neox_layers_1_post_attention_layernorm_bias2, alloc352) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias2) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[30] storage15: R.Object = R.vm.alloc_storage(R.shape([8192]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc353: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2, alloc352, gpt_neox_layers_1_mlp_dense_h_to_4h_bias2, alloc353) R.vm.kill_object(alloc352) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias2) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[33] alloc354: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2, alloc353, gpt_neox_layers_1_mlp_dense_4h_to_h_bias2, alloc354) R.vm.kill_object(alloc353) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias2) gpt_neox_layers_1_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[27] alloc355: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_1_attention_dense_q_weight2, gpt_neox_layers_1_attention_dense_q_scale2, lv299, gpt_neox_layers_1_attention_dense_bias2, alloc354, alloc348, alloc355) R.vm.kill_object(alloc348) R.vm.kill_object(lv299) R.vm.kill_object(alloc354) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias2) gpt_neox_layers_2_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[35] alloc356: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc355, gpt_neox_layers_2_input_layernorm_weight2, gpt_neox_layers_2_input_layernorm_bias2, alloc356) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias2) gpt_neox_layers_2_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[40] alloc357: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_2_attention_query_key_value_q_weight2, gpt_neox_layers_2_attention_query_key_value_q_scale2, alloc356, gpt_neox_layers_2_attention_query_key_value_bias2, alloc357) R.vm.kill_object(alloc356) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias2) lv304: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc357, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc357) alloc358: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _358: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), lv304, alloc358) R.vm.kill_object(lv304) lv305: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc358, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc358) gpt_neox_layers_2_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[37] alloc359: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc355, gpt_neox_layers_2_post_attention_layernorm_weight2, gpt_neox_layers_2_post_attention_layernorm_bias2, alloc359) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias2) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[46] alloc360: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2, alloc359, gpt_neox_layers_2_mlp_dense_h_to_4h_bias2, alloc360) R.vm.kill_object(alloc359) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias2) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[49] alloc361: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2, alloc360, gpt_neox_layers_2_mlp_dense_4h_to_h_bias2, alloc361) R.vm.kill_object(alloc360) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias2) gpt_neox_layers_2_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[43] alloc362: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_2_attention_dense_q_weight2, gpt_neox_layers_2_attention_dense_q_scale2, lv305, gpt_neox_layers_2_attention_dense_bias2, alloc361, alloc355, alloc362) R.vm.kill_object(alloc355) R.vm.kill_object(lv305) R.vm.kill_object(alloc361) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias2) gpt_neox_layers_3_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[51] alloc363: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc362, gpt_neox_layers_3_input_layernorm_weight2, gpt_neox_layers_3_input_layernorm_bias2, alloc363) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias2) gpt_neox_layers_3_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[56] alloc364: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_3_attention_query_key_value_q_weight2, gpt_neox_layers_3_attention_query_key_value_q_scale2, alloc363, gpt_neox_layers_3_attention_query_key_value_bias2, alloc364) R.vm.kill_object(alloc363) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias2) lv310: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc364, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc364) alloc365: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _365: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), lv310, alloc365) R.vm.kill_object(lv310) lv311: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc365, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc365) gpt_neox_layers_3_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[53] alloc366: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc362, gpt_neox_layers_3_post_attention_layernorm_weight2, gpt_neox_layers_3_post_attention_layernorm_bias2, alloc366) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias2) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[62] alloc367: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2, alloc366, gpt_neox_layers_3_mlp_dense_h_to_4h_bias2, alloc367) R.vm.kill_object(alloc366) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias2) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[65] alloc368: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2, alloc367, gpt_neox_layers_3_mlp_dense_4h_to_h_bias2, alloc368) R.vm.kill_object(alloc367) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias2) gpt_neox_layers_3_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[59] alloc369: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_3_attention_dense_q_weight2, gpt_neox_layers_3_attention_dense_q_scale2, lv311, gpt_neox_layers_3_attention_dense_bias2, alloc368, alloc362, alloc369) R.vm.kill_object(alloc362) R.vm.kill_object(lv311) R.vm.kill_object(alloc368) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias2) gpt_neox_layers_4_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[67] alloc370: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc369, gpt_neox_layers_4_input_layernorm_weight2, gpt_neox_layers_4_input_layernorm_bias2, alloc370) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias2) gpt_neox_layers_4_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[72] alloc371: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_4_attention_query_key_value_q_weight2, gpt_neox_layers_4_attention_query_key_value_q_scale2, alloc370, gpt_neox_layers_4_attention_query_key_value_bias2, alloc371) R.vm.kill_object(alloc370) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias2) lv316: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc371, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc371) alloc372: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _372: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), lv316, alloc372) R.vm.kill_object(lv316) lv317: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc372, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc372) gpt_neox_layers_4_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[69] alloc373: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc369, gpt_neox_layers_4_post_attention_layernorm_weight2, gpt_neox_layers_4_post_attention_layernorm_bias2, alloc373) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias2) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[78] alloc374: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2, alloc373, gpt_neox_layers_4_mlp_dense_h_to_4h_bias2, alloc374) R.vm.kill_object(alloc373) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias2) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[81] alloc375: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2, alloc374, gpt_neox_layers_4_mlp_dense_4h_to_h_bias2, alloc375) R.vm.kill_object(alloc374) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias2) gpt_neox_layers_4_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[75] alloc376: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_4_attention_dense_q_weight2, gpt_neox_layers_4_attention_dense_q_scale2, lv317, gpt_neox_layers_4_attention_dense_bias2, alloc375, alloc369, alloc376) R.vm.kill_object(alloc369) R.vm.kill_object(lv317) R.vm.kill_object(alloc375) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias2) gpt_neox_layers_5_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[83] alloc377: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc376, gpt_neox_layers_5_input_layernorm_weight2, gpt_neox_layers_5_input_layernorm_bias2, alloc377) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias2) gpt_neox_layers_5_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[88] alloc378: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_5_attention_query_key_value_q_weight2, gpt_neox_layers_5_attention_query_key_value_q_scale2, alloc377, gpt_neox_layers_5_attention_query_key_value_bias2, alloc378) R.vm.kill_object(alloc377) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias2) lv322: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc378, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc378) alloc379: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _379: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), lv322, alloc379) R.vm.kill_object(lv322) lv323: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc379, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc379) gpt_neox_layers_5_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[85] alloc380: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc376, gpt_neox_layers_5_post_attention_layernorm_weight2, gpt_neox_layers_5_post_attention_layernorm_bias2, alloc380) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias2) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[94] alloc381: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2, alloc380, gpt_neox_layers_5_mlp_dense_h_to_4h_bias2, alloc381) R.vm.kill_object(alloc380) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias2) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[97] alloc382: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2, alloc381, gpt_neox_layers_5_mlp_dense_4h_to_h_bias2, alloc382) R.vm.kill_object(alloc381) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias2) gpt_neox_layers_5_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[91] alloc383: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_5_attention_dense_q_weight2, gpt_neox_layers_5_attention_dense_q_scale2, lv323, gpt_neox_layers_5_attention_dense_bias2, alloc382, alloc376, alloc383) R.vm.kill_object(alloc376) R.vm.kill_object(lv323) R.vm.kill_object(alloc382) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias2) gpt_neox_layers_6_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[98] gpt_neox_layers_6_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[99] alloc384: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc383, gpt_neox_layers_6_input_layernorm_weight2, gpt_neox_layers_6_input_layernorm_bias2, alloc384) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_bias2) gpt_neox_layers_6_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[102] gpt_neox_layers_6_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[103] gpt_neox_layers_6_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[104] alloc385: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_6_attention_query_key_value_q_weight2, gpt_neox_layers_6_attention_query_key_value_q_scale2, alloc384, gpt_neox_layers_6_attention_query_key_value_bias2, alloc385) R.vm.kill_object(alloc384) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_bias2) lv328: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc385, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc385) alloc386: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _386: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), lv328, alloc386) R.vm.kill_object(lv328) lv329: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc386, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc386) gpt_neox_layers_6_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[100] gpt_neox_layers_6_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[101] alloc387: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc383, gpt_neox_layers_6_post_attention_layernorm_weight2, gpt_neox_layers_6_post_attention_layernorm_bias2, alloc387) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_bias2) gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[108] gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[109] gpt_neox_layers_6_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[110] alloc388: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale2, alloc387, gpt_neox_layers_6_mlp_dense_h_to_4h_bias2, alloc388) R.vm.kill_object(alloc387) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_bias2) gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[111] gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[112] gpt_neox_layers_6_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[113] alloc389: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale2, alloc388, gpt_neox_layers_6_mlp_dense_4h_to_h_bias2, alloc389) R.vm.kill_object(alloc388) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_bias2) gpt_neox_layers_6_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[105] gpt_neox_layers_6_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[106] gpt_neox_layers_6_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[107] alloc390: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_6_attention_dense_q_weight2, gpt_neox_layers_6_attention_dense_q_scale2, lv329, gpt_neox_layers_6_attention_dense_bias2, alloc389, alloc383, alloc390) R.vm.kill_object(alloc383) R.vm.kill_object(lv329) R.vm.kill_object(alloc389) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_6_attention_dense_bias2) gpt_neox_layers_7_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[114] gpt_neox_layers_7_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[115] alloc391: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc390, gpt_neox_layers_7_input_layernorm_weight2, gpt_neox_layers_7_input_layernorm_bias2, alloc391) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_bias2) gpt_neox_layers_7_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[118] gpt_neox_layers_7_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[119] gpt_neox_layers_7_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[120] alloc392: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_7_attention_query_key_value_q_weight2, gpt_neox_layers_7_attention_query_key_value_q_scale2, alloc391, gpt_neox_layers_7_attention_query_key_value_bias2, alloc392) R.vm.kill_object(alloc391) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_bias2) lv334: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc392, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc392) alloc393: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _393: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), lv334, alloc393) R.vm.kill_object(lv334) lv335: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc393, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc393) gpt_neox_layers_7_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[116] gpt_neox_layers_7_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[117] alloc394: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc390, gpt_neox_layers_7_post_attention_layernorm_weight2, gpt_neox_layers_7_post_attention_layernorm_bias2, alloc394) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_bias2) gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[124] gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[125] gpt_neox_layers_7_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[126] alloc395: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale2, alloc394, gpt_neox_layers_7_mlp_dense_h_to_4h_bias2, alloc395) R.vm.kill_object(alloc394) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_bias2) gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[127] gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[128] gpt_neox_layers_7_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[129] alloc396: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale2, alloc395, gpt_neox_layers_7_mlp_dense_4h_to_h_bias2, alloc396) R.vm.kill_object(alloc395) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_bias2) gpt_neox_layers_7_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[121] gpt_neox_layers_7_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[122] gpt_neox_layers_7_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[123] alloc397: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_7_attention_dense_q_weight2, gpt_neox_layers_7_attention_dense_q_scale2, lv335, gpt_neox_layers_7_attention_dense_bias2, alloc396, alloc390, alloc397) R.vm.kill_object(alloc390) R.vm.kill_object(lv335) R.vm.kill_object(alloc396) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_7_attention_dense_bias2) gpt_neox_layers_8_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[130] gpt_neox_layers_8_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[131] alloc398: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc397, gpt_neox_layers_8_input_layernorm_weight2, gpt_neox_layers_8_input_layernorm_bias2, alloc398) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_bias2) gpt_neox_layers_8_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[134] gpt_neox_layers_8_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[135] gpt_neox_layers_8_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[136] alloc399: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_8_attention_query_key_value_q_weight2, gpt_neox_layers_8_attention_query_key_value_q_scale2, alloc398, gpt_neox_layers_8_attention_query_key_value_bias2, alloc399) R.vm.kill_object(alloc398) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_bias2) lv340: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc399, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc399) alloc400: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _400: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), lv340, alloc400) R.vm.kill_object(lv340) lv341: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc400, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc400) gpt_neox_layers_8_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[132] gpt_neox_layers_8_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[133] alloc401: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc397, gpt_neox_layers_8_post_attention_layernorm_weight2, gpt_neox_layers_8_post_attention_layernorm_bias2, alloc401) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_bias2) gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[140] gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[141] gpt_neox_layers_8_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[142] alloc402: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale2, alloc401, gpt_neox_layers_8_mlp_dense_h_to_4h_bias2, alloc402) R.vm.kill_object(alloc401) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_bias2) gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[143] gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[144] gpt_neox_layers_8_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[145] alloc403: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale2, alloc402, gpt_neox_layers_8_mlp_dense_4h_to_h_bias2, alloc403) R.vm.kill_object(alloc402) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_bias2) gpt_neox_layers_8_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[137] gpt_neox_layers_8_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[138] gpt_neox_layers_8_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[139] alloc404: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_8_attention_dense_q_weight2, gpt_neox_layers_8_attention_dense_q_scale2, lv341, gpt_neox_layers_8_attention_dense_bias2, alloc403, alloc397, alloc404) R.vm.kill_object(alloc397) R.vm.kill_object(lv341) R.vm.kill_object(alloc403) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_8_attention_dense_bias2) gpt_neox_layers_9_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[146] gpt_neox_layers_9_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[147] alloc405: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc404, gpt_neox_layers_9_input_layernorm_weight2, gpt_neox_layers_9_input_layernorm_bias2, alloc405) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_bias2) gpt_neox_layers_9_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[150] gpt_neox_layers_9_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[151] gpt_neox_layers_9_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[152] alloc406: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_9_attention_query_key_value_q_weight2, gpt_neox_layers_9_attention_query_key_value_q_scale2, alloc405, gpt_neox_layers_9_attention_query_key_value_bias2, alloc406) R.vm.kill_object(alloc405) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_bias2) lv346: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc406, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc406) alloc407: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _407: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), lv346, alloc407) R.vm.kill_object(lv346) lv347: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc407, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc407) gpt_neox_layers_9_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[148] gpt_neox_layers_9_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[149] alloc408: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc404, gpt_neox_layers_9_post_attention_layernorm_weight2, gpt_neox_layers_9_post_attention_layernorm_bias2, alloc408) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_bias2) gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[156] gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[157] gpt_neox_layers_9_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[158] alloc409: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale2, alloc408, gpt_neox_layers_9_mlp_dense_h_to_4h_bias2, alloc409) R.vm.kill_object(alloc408) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_bias2) gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[159] gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[160] gpt_neox_layers_9_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[161] alloc410: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale2, alloc409, gpt_neox_layers_9_mlp_dense_4h_to_h_bias2, alloc410) R.vm.kill_object(alloc409) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_bias2) gpt_neox_layers_9_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[153] gpt_neox_layers_9_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[154] gpt_neox_layers_9_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[155] alloc411: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_9_attention_dense_q_weight2, gpt_neox_layers_9_attention_dense_q_scale2, lv347, gpt_neox_layers_9_attention_dense_bias2, alloc410, alloc404, alloc411) R.vm.kill_object(alloc404) R.vm.kill_object(lv347) R.vm.kill_object(alloc410) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_9_attention_dense_bias2) gpt_neox_layers_10_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[162] gpt_neox_layers_10_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[163] alloc412: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc411, gpt_neox_layers_10_input_layernorm_weight2, gpt_neox_layers_10_input_layernorm_bias2, alloc412) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_bias2) gpt_neox_layers_10_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[166] gpt_neox_layers_10_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[167] gpt_neox_layers_10_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[168] alloc413: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_10_attention_query_key_value_q_weight2, gpt_neox_layers_10_attention_query_key_value_q_scale2, alloc412, gpt_neox_layers_10_attention_query_key_value_bias2, alloc413) R.vm.kill_object(alloc412) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_bias2) lv352: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc413, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc413) alloc414: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _414: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), lv352, alloc414) R.vm.kill_object(lv352) lv353: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc414, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc414) gpt_neox_layers_10_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[164] gpt_neox_layers_10_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[165] alloc415: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc411, gpt_neox_layers_10_post_attention_layernorm_weight2, gpt_neox_layers_10_post_attention_layernorm_bias2, alloc415) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_bias2) gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[172] gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[173] gpt_neox_layers_10_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[174] alloc416: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale2, alloc415, gpt_neox_layers_10_mlp_dense_h_to_4h_bias2, alloc416) R.vm.kill_object(alloc415) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_bias2) gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[175] gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[176] gpt_neox_layers_10_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[177] alloc417: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale2, alloc416, gpt_neox_layers_10_mlp_dense_4h_to_h_bias2, alloc417) R.vm.kill_object(alloc416) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_bias2) gpt_neox_layers_10_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[169] gpt_neox_layers_10_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[170] gpt_neox_layers_10_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[171] alloc418: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_10_attention_dense_q_weight2, gpt_neox_layers_10_attention_dense_q_scale2, lv353, gpt_neox_layers_10_attention_dense_bias2, alloc417, alloc411, alloc418) R.vm.kill_object(alloc411) R.vm.kill_object(lv353) R.vm.kill_object(alloc417) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_10_attention_dense_bias2) gpt_neox_layers_11_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[178] gpt_neox_layers_11_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[179] alloc419: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc418, gpt_neox_layers_11_input_layernorm_weight2, gpt_neox_layers_11_input_layernorm_bias2, alloc419) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_bias2) gpt_neox_layers_11_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[182] gpt_neox_layers_11_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[183] gpt_neox_layers_11_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[184] alloc420: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_11_attention_query_key_value_q_weight2, gpt_neox_layers_11_attention_query_key_value_q_scale2, alloc419, gpt_neox_layers_11_attention_query_key_value_bias2, alloc420) R.vm.kill_object(alloc419) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_bias2) lv358: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc420, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc420) alloc421: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _421: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), lv358, alloc421) R.vm.kill_object(lv358) lv359: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc421, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc421) gpt_neox_layers_11_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[180] gpt_neox_layers_11_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[181] alloc422: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc418, gpt_neox_layers_11_post_attention_layernorm_weight2, gpt_neox_layers_11_post_attention_layernorm_bias2, alloc422) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_bias2) gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[188] gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[189] gpt_neox_layers_11_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[190] alloc423: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale2, alloc422, gpt_neox_layers_11_mlp_dense_h_to_4h_bias2, alloc423) R.vm.kill_object(alloc422) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_bias2) gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[191] gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[192] gpt_neox_layers_11_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[193] alloc424: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale2, alloc423, gpt_neox_layers_11_mlp_dense_4h_to_h_bias2, alloc424) R.vm.kill_object(alloc423) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_bias2) gpt_neox_layers_11_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[185] gpt_neox_layers_11_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[186] gpt_neox_layers_11_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[187] alloc425: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_11_attention_dense_q_weight2, gpt_neox_layers_11_attention_dense_q_scale2, lv359, gpt_neox_layers_11_attention_dense_bias2, alloc424, alloc418, alloc425) R.vm.kill_object(alloc418) R.vm.kill_object(lv359) R.vm.kill_object(alloc424) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_11_attention_dense_bias2) gpt_neox_layers_12_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[194] gpt_neox_layers_12_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[195] alloc426: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc425, gpt_neox_layers_12_input_layernorm_weight2, gpt_neox_layers_12_input_layernorm_bias2, alloc426) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_bias2) gpt_neox_layers_12_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[198] gpt_neox_layers_12_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[199] gpt_neox_layers_12_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[200] alloc427: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_12_attention_query_key_value_q_weight2, gpt_neox_layers_12_attention_query_key_value_q_scale2, alloc426, gpt_neox_layers_12_attention_query_key_value_bias2, alloc427) R.vm.kill_object(alloc426) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_bias2) lv364: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc427, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc427) alloc428: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _428: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), lv364, alloc428) R.vm.kill_object(lv364) lv365: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc428, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc428) gpt_neox_layers_12_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[196] gpt_neox_layers_12_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[197] alloc429: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc425, gpt_neox_layers_12_post_attention_layernorm_weight2, gpt_neox_layers_12_post_attention_layernorm_bias2, alloc429) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_bias2) gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[204] gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[205] gpt_neox_layers_12_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[206] alloc430: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale2, alloc429, gpt_neox_layers_12_mlp_dense_h_to_4h_bias2, alloc430) R.vm.kill_object(alloc429) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_bias2) gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[207] gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[208] gpt_neox_layers_12_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[209] alloc431: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale2, alloc430, gpt_neox_layers_12_mlp_dense_4h_to_h_bias2, alloc431) R.vm.kill_object(alloc430) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_bias2) gpt_neox_layers_12_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[201] gpt_neox_layers_12_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[202] gpt_neox_layers_12_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[203] alloc432: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_12_attention_dense_q_weight2, gpt_neox_layers_12_attention_dense_q_scale2, lv365, gpt_neox_layers_12_attention_dense_bias2, alloc431, alloc425, alloc432) R.vm.kill_object(alloc425) R.vm.kill_object(lv365) R.vm.kill_object(alloc431) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_12_attention_dense_bias2) gpt_neox_layers_13_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[210] gpt_neox_layers_13_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[211] alloc433: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc432, gpt_neox_layers_13_input_layernorm_weight2, gpt_neox_layers_13_input_layernorm_bias2, alloc433) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_bias2) gpt_neox_layers_13_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[214] gpt_neox_layers_13_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[215] gpt_neox_layers_13_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[216] alloc434: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_13_attention_query_key_value_q_weight2, gpt_neox_layers_13_attention_query_key_value_q_scale2, alloc433, gpt_neox_layers_13_attention_query_key_value_bias2, alloc434) R.vm.kill_object(alloc433) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_bias2) lv370: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc434, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc434) alloc435: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _435: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), lv370, alloc435) R.vm.kill_object(lv370) lv371: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc435, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc435) gpt_neox_layers_13_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[212] gpt_neox_layers_13_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[213] alloc436: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc432, gpt_neox_layers_13_post_attention_layernorm_weight2, gpt_neox_layers_13_post_attention_layernorm_bias2, alloc436) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_bias2) gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[220] gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[221] gpt_neox_layers_13_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[222] alloc437: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale2, alloc436, gpt_neox_layers_13_mlp_dense_h_to_4h_bias2, alloc437) R.vm.kill_object(alloc436) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_bias2) gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[223] gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[224] gpt_neox_layers_13_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[225] alloc438: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale2, alloc437, gpt_neox_layers_13_mlp_dense_4h_to_h_bias2, alloc438) R.vm.kill_object(alloc437) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_bias2) gpt_neox_layers_13_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[217] gpt_neox_layers_13_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[218] gpt_neox_layers_13_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[219] alloc439: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_13_attention_dense_q_weight2, gpt_neox_layers_13_attention_dense_q_scale2, lv371, gpt_neox_layers_13_attention_dense_bias2, alloc438, alloc432, alloc439) R.vm.kill_object(alloc432) R.vm.kill_object(lv371) R.vm.kill_object(alloc438) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_13_attention_dense_bias2) gpt_neox_layers_14_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[226] gpt_neox_layers_14_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[227] alloc440: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc439, gpt_neox_layers_14_input_layernorm_weight2, gpt_neox_layers_14_input_layernorm_bias2, alloc440) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_bias2) gpt_neox_layers_14_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[230] gpt_neox_layers_14_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[231] gpt_neox_layers_14_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[232] alloc441: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_14_attention_query_key_value_q_weight2, gpt_neox_layers_14_attention_query_key_value_q_scale2, alloc440, gpt_neox_layers_14_attention_query_key_value_bias2, alloc441) R.vm.kill_object(alloc440) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_bias2) lv376: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc441, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc441) alloc442: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _442: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), lv376, alloc442) R.vm.kill_object(lv376) lv377: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc442, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc442) gpt_neox_layers_14_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[228] gpt_neox_layers_14_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[229] alloc443: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc439, gpt_neox_layers_14_post_attention_layernorm_weight2, gpt_neox_layers_14_post_attention_layernorm_bias2, alloc443) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_bias2) gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[236] gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[237] gpt_neox_layers_14_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[238] alloc444: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale2, alloc443, gpt_neox_layers_14_mlp_dense_h_to_4h_bias2, alloc444) R.vm.kill_object(alloc443) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_bias2) gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[239] gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[240] gpt_neox_layers_14_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[241] alloc445: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale2, alloc444, gpt_neox_layers_14_mlp_dense_4h_to_h_bias2, alloc445) R.vm.kill_object(alloc444) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_bias2) gpt_neox_layers_14_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[233] gpt_neox_layers_14_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[234] gpt_neox_layers_14_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[235] alloc446: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_14_attention_dense_q_weight2, gpt_neox_layers_14_attention_dense_q_scale2, lv377, gpt_neox_layers_14_attention_dense_bias2, alloc445, alloc439, alloc446) R.vm.kill_object(alloc439) R.vm.kill_object(lv377) R.vm.kill_object(alloc445) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_14_attention_dense_bias2) gpt_neox_layers_15_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[242] gpt_neox_layers_15_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[243] alloc447: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc446, gpt_neox_layers_15_input_layernorm_weight2, gpt_neox_layers_15_input_layernorm_bias2, alloc447) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_bias2) gpt_neox_layers_15_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[246] gpt_neox_layers_15_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[247] gpt_neox_layers_15_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[248] alloc448: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_15_attention_query_key_value_q_weight2, gpt_neox_layers_15_attention_query_key_value_q_scale2, alloc447, gpt_neox_layers_15_attention_query_key_value_bias2, alloc448) R.vm.kill_object(alloc447) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_bias2) lv382: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc448, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc448) alloc449: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _449: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), lv382, alloc449) R.vm.kill_object(lv382) lv383: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc449, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc449) gpt_neox_layers_15_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[244] gpt_neox_layers_15_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[245] alloc450: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc446, gpt_neox_layers_15_post_attention_layernorm_weight2, gpt_neox_layers_15_post_attention_layernorm_bias2, alloc450) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_bias2) gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[252] gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[253] gpt_neox_layers_15_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[254] alloc451: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale2, alloc450, gpt_neox_layers_15_mlp_dense_h_to_4h_bias2, alloc451) R.vm.kill_object(alloc450) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_bias2) gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[255] gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[256] gpt_neox_layers_15_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[257] alloc452: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale2, alloc451, gpt_neox_layers_15_mlp_dense_4h_to_h_bias2, alloc452) R.vm.kill_object(alloc451) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_bias2) gpt_neox_layers_15_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[249] gpt_neox_layers_15_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[250] gpt_neox_layers_15_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[251] alloc453: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_15_attention_dense_q_weight2, gpt_neox_layers_15_attention_dense_q_scale2, lv383, gpt_neox_layers_15_attention_dense_bias2, alloc452, alloc446, alloc453) R.vm.kill_object(alloc446) R.vm.kill_object(lv383) R.vm.kill_object(alloc452) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_15_attention_dense_bias2) gpt_neox_layers_16_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[258] gpt_neox_layers_16_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[259] alloc454: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc453, gpt_neox_layers_16_input_layernorm_weight2, gpt_neox_layers_16_input_layernorm_bias2, alloc454) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_bias2) gpt_neox_layers_16_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[262] gpt_neox_layers_16_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[263] gpt_neox_layers_16_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[264] alloc455: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_16_attention_query_key_value_q_weight2, gpt_neox_layers_16_attention_query_key_value_q_scale2, alloc454, gpt_neox_layers_16_attention_query_key_value_bias2, alloc455) R.vm.kill_object(alloc454) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_bias2) lv388: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc455, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc455) alloc456: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _456: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), lv388, alloc456) R.vm.kill_object(lv388) lv389: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc456, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc456) gpt_neox_layers_16_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[260] gpt_neox_layers_16_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[261] alloc457: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc453, gpt_neox_layers_16_post_attention_layernorm_weight2, gpt_neox_layers_16_post_attention_layernorm_bias2, alloc457) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_bias2) gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[268] gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[269] gpt_neox_layers_16_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[270] alloc458: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale2, alloc457, gpt_neox_layers_16_mlp_dense_h_to_4h_bias2, alloc458) R.vm.kill_object(alloc457) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_bias2) gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[271] gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[272] gpt_neox_layers_16_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[273] alloc459: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale2, alloc458, gpt_neox_layers_16_mlp_dense_4h_to_h_bias2, alloc459) R.vm.kill_object(alloc458) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_bias2) gpt_neox_layers_16_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[265] gpt_neox_layers_16_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[266] gpt_neox_layers_16_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[267] alloc460: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_16_attention_dense_q_weight2, gpt_neox_layers_16_attention_dense_q_scale2, lv389, gpt_neox_layers_16_attention_dense_bias2, alloc459, alloc453, alloc460) R.vm.kill_object(alloc453) R.vm.kill_object(lv389) R.vm.kill_object(alloc459) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_16_attention_dense_bias2) gpt_neox_layers_17_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[274] gpt_neox_layers_17_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[275] alloc461: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc460, gpt_neox_layers_17_input_layernorm_weight2, gpt_neox_layers_17_input_layernorm_bias2, alloc461) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_bias2) gpt_neox_layers_17_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[278] gpt_neox_layers_17_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[279] gpt_neox_layers_17_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[280] alloc462: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_17_attention_query_key_value_q_weight2, gpt_neox_layers_17_attention_query_key_value_q_scale2, alloc461, gpt_neox_layers_17_attention_query_key_value_bias2, alloc462) R.vm.kill_object(alloc461) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_bias2) lv394: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc462, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc462) alloc463: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _463: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), lv394, alloc463) R.vm.kill_object(lv394) lv395: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc463, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc463) gpt_neox_layers_17_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[276] gpt_neox_layers_17_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[277] alloc464: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc460, gpt_neox_layers_17_post_attention_layernorm_weight2, gpt_neox_layers_17_post_attention_layernorm_bias2, alloc464) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_bias2) gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[284] gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[285] gpt_neox_layers_17_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[286] alloc465: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale2, alloc464, gpt_neox_layers_17_mlp_dense_h_to_4h_bias2, alloc465) R.vm.kill_object(alloc464) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_bias2) gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[287] gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[288] gpt_neox_layers_17_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[289] alloc466: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale2, alloc465, gpt_neox_layers_17_mlp_dense_4h_to_h_bias2, alloc466) R.vm.kill_object(alloc465) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_bias2) gpt_neox_layers_17_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[281] gpt_neox_layers_17_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[282] gpt_neox_layers_17_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[283] alloc467: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_17_attention_dense_q_weight2, gpt_neox_layers_17_attention_dense_q_scale2, lv395, gpt_neox_layers_17_attention_dense_bias2, alloc466, alloc460, alloc467) R.vm.kill_object(alloc460) R.vm.kill_object(lv395) R.vm.kill_object(alloc466) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_17_attention_dense_bias2) gpt_neox_layers_18_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[290] gpt_neox_layers_18_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[291] alloc468: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc467, gpt_neox_layers_18_input_layernorm_weight2, gpt_neox_layers_18_input_layernorm_bias2, alloc468) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_bias2) gpt_neox_layers_18_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[294] gpt_neox_layers_18_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[295] gpt_neox_layers_18_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[296] alloc469: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_18_attention_query_key_value_q_weight2, gpt_neox_layers_18_attention_query_key_value_q_scale2, alloc468, gpt_neox_layers_18_attention_query_key_value_bias2, alloc469) R.vm.kill_object(alloc468) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_bias2) lv400: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc469, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc469) alloc470: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _470: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), lv400, alloc470) R.vm.kill_object(lv400) lv401: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc470, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc470) gpt_neox_layers_18_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[292] gpt_neox_layers_18_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[293] alloc471: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc467, gpt_neox_layers_18_post_attention_layernorm_weight2, gpt_neox_layers_18_post_attention_layernorm_bias2, alloc471) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_bias2) gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[300] gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[301] gpt_neox_layers_18_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[302] alloc472: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale2, alloc471, gpt_neox_layers_18_mlp_dense_h_to_4h_bias2, alloc472) R.vm.kill_object(alloc471) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_bias2) gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[303] gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[304] gpt_neox_layers_18_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[305] alloc473: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale2, alloc472, gpt_neox_layers_18_mlp_dense_4h_to_h_bias2, alloc473) R.vm.kill_object(alloc472) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_bias2) gpt_neox_layers_18_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[297] gpt_neox_layers_18_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[298] gpt_neox_layers_18_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[299] alloc474: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_18_attention_dense_q_weight2, gpt_neox_layers_18_attention_dense_q_scale2, lv401, gpt_neox_layers_18_attention_dense_bias2, alloc473, alloc467, alloc474) R.vm.kill_object(alloc467) R.vm.kill_object(lv401) R.vm.kill_object(alloc473) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_18_attention_dense_bias2) gpt_neox_layers_19_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[306] gpt_neox_layers_19_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[307] alloc475: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc474, gpt_neox_layers_19_input_layernorm_weight2, gpt_neox_layers_19_input_layernorm_bias2, alloc475) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_bias2) gpt_neox_layers_19_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[310] gpt_neox_layers_19_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[311] gpt_neox_layers_19_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[312] alloc476: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_19_attention_query_key_value_q_weight2, gpt_neox_layers_19_attention_query_key_value_q_scale2, alloc475, gpt_neox_layers_19_attention_query_key_value_bias2, alloc476) R.vm.kill_object(alloc475) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_bias2) lv406: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc476, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc476) alloc477: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _477: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), lv406, alloc477) R.vm.kill_object(lv406) lv407: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc477, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc477) gpt_neox_layers_19_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[308] gpt_neox_layers_19_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[309] alloc478: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc474, gpt_neox_layers_19_post_attention_layernorm_weight2, gpt_neox_layers_19_post_attention_layernorm_bias2, alloc478) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_bias2) gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[316] gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[317] gpt_neox_layers_19_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[318] alloc479: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale2, alloc478, gpt_neox_layers_19_mlp_dense_h_to_4h_bias2, alloc479) R.vm.kill_object(alloc478) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_bias2) gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[319] gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[320] gpt_neox_layers_19_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[321] alloc480: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale2, alloc479, gpt_neox_layers_19_mlp_dense_4h_to_h_bias2, alloc480) R.vm.kill_object(alloc479) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_bias2) gpt_neox_layers_19_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[313] gpt_neox_layers_19_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[314] gpt_neox_layers_19_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[315] alloc481: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_19_attention_dense_q_weight2, gpt_neox_layers_19_attention_dense_q_scale2, lv407, gpt_neox_layers_19_attention_dense_bias2, alloc480, alloc474, alloc481) R.vm.kill_object(alloc474) R.vm.kill_object(lv407) R.vm.kill_object(alloc480) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_19_attention_dense_bias2) gpt_neox_layers_20_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[322] gpt_neox_layers_20_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[323] alloc482: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc481, gpt_neox_layers_20_input_layernorm_weight2, gpt_neox_layers_20_input_layernorm_bias2, alloc482) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_bias2) gpt_neox_layers_20_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[326] gpt_neox_layers_20_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[327] gpt_neox_layers_20_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[328] alloc483: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_20_attention_query_key_value_q_weight2, gpt_neox_layers_20_attention_query_key_value_q_scale2, alloc482, gpt_neox_layers_20_attention_query_key_value_bias2, alloc483) R.vm.kill_object(alloc482) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_bias2) lv412: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc483, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc483) alloc484: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _484: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), lv412, alloc484) R.vm.kill_object(lv412) lv413: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc484, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc484) gpt_neox_layers_20_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[324] gpt_neox_layers_20_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[325] alloc485: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc481, gpt_neox_layers_20_post_attention_layernorm_weight2, gpt_neox_layers_20_post_attention_layernorm_bias2, alloc485) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_bias2) gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[332] gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[333] gpt_neox_layers_20_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[334] alloc486: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale2, alloc485, gpt_neox_layers_20_mlp_dense_h_to_4h_bias2, alloc486) R.vm.kill_object(alloc485) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_bias2) gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[335] gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[336] gpt_neox_layers_20_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[337] alloc487: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale2, alloc486, gpt_neox_layers_20_mlp_dense_4h_to_h_bias2, alloc487) R.vm.kill_object(alloc486) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_bias2) gpt_neox_layers_20_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[329] gpt_neox_layers_20_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[330] gpt_neox_layers_20_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[331] alloc488: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_20_attention_dense_q_weight2, gpt_neox_layers_20_attention_dense_q_scale2, lv413, gpt_neox_layers_20_attention_dense_bias2, alloc487, alloc481, alloc488) R.vm.kill_object(alloc481) R.vm.kill_object(lv413) R.vm.kill_object(alloc487) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_20_attention_dense_bias2) gpt_neox_layers_21_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[338] gpt_neox_layers_21_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[339] alloc489: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc488, gpt_neox_layers_21_input_layernorm_weight2, gpt_neox_layers_21_input_layernorm_bias2, alloc489) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_bias2) gpt_neox_layers_21_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[342] gpt_neox_layers_21_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[343] gpt_neox_layers_21_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[344] alloc490: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_21_attention_query_key_value_q_weight2, gpt_neox_layers_21_attention_query_key_value_q_scale2, alloc489, gpt_neox_layers_21_attention_query_key_value_bias2, alloc490) R.vm.kill_object(alloc489) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_bias2) lv418: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc490, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc490) alloc491: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _491: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), lv418, alloc491) R.vm.kill_object(lv418) lv419: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc491, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc491) gpt_neox_layers_21_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[340] gpt_neox_layers_21_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[341] alloc492: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc488, gpt_neox_layers_21_post_attention_layernorm_weight2, gpt_neox_layers_21_post_attention_layernorm_bias2, alloc492) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_bias2) gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[348] gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[349] gpt_neox_layers_21_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[350] alloc493: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale2, alloc492, gpt_neox_layers_21_mlp_dense_h_to_4h_bias2, alloc493) R.vm.kill_object(alloc492) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_bias2) gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[351] gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[352] gpt_neox_layers_21_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[353] alloc494: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale2, alloc493, gpt_neox_layers_21_mlp_dense_4h_to_h_bias2, alloc494) R.vm.kill_object(alloc493) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_bias2) gpt_neox_layers_21_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[345] gpt_neox_layers_21_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[346] gpt_neox_layers_21_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[347] alloc495: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_21_attention_dense_q_weight2, gpt_neox_layers_21_attention_dense_q_scale2, lv419, gpt_neox_layers_21_attention_dense_bias2, alloc494, alloc488, alloc495) R.vm.kill_object(alloc488) R.vm.kill_object(lv419) R.vm.kill_object(alloc494) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_21_attention_dense_bias2) gpt_neox_layers_22_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[354] gpt_neox_layers_22_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[355] alloc496: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc495, gpt_neox_layers_22_input_layernorm_weight2, gpt_neox_layers_22_input_layernorm_bias2, alloc496) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_bias2) gpt_neox_layers_22_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[358] gpt_neox_layers_22_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[359] gpt_neox_layers_22_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[360] alloc497: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_22_attention_query_key_value_q_weight2, gpt_neox_layers_22_attention_query_key_value_q_scale2, alloc496, gpt_neox_layers_22_attention_query_key_value_bias2, alloc497) R.vm.kill_object(alloc496) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_bias2) lv424: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc497, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc497) alloc498: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _498: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), lv424, alloc498) R.vm.kill_object(lv424) lv425: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc498, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc498) gpt_neox_layers_22_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[356] gpt_neox_layers_22_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[357] alloc499: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc495, gpt_neox_layers_22_post_attention_layernorm_weight2, gpt_neox_layers_22_post_attention_layernorm_bias2, alloc499) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_bias2) gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[364] gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[365] gpt_neox_layers_22_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[366] alloc500: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale2, alloc499, gpt_neox_layers_22_mlp_dense_h_to_4h_bias2, alloc500) R.vm.kill_object(alloc499) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_bias2) gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[367] gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[368] gpt_neox_layers_22_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[369] alloc501: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale2, alloc500, gpt_neox_layers_22_mlp_dense_4h_to_h_bias2, alloc501) R.vm.kill_object(alloc500) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_bias2) gpt_neox_layers_22_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[361] gpt_neox_layers_22_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[362] gpt_neox_layers_22_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[363] alloc502: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage14, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) R.vm.kill_object(storage14) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_22_attention_dense_q_weight2, gpt_neox_layers_22_attention_dense_q_scale2, lv425, gpt_neox_layers_22_attention_dense_bias2, alloc501, alloc495, alloc502) R.vm.kill_object(alloc495) R.vm.kill_object(lv425) R.vm.kill_object(alloc501) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_22_attention_dense_bias2) gpt_neox_layers_23_input_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[370] gpt_neox_layers_23_input_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[371] alloc503: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc502, gpt_neox_layers_23_input_layernorm_weight2, gpt_neox_layers_23_input_layernorm_bias2, alloc503) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_bias2) gpt_neox_layers_23_attention_query_key_value_q_weight2: R.Tensor((3072, 128), dtype="uint32") = packed_params[374] gpt_neox_layers_23_attention_query_key_value_q_scale2: R.Tensor((3072, 32), dtype="float16") = packed_params[375] gpt_neox_layers_23_attention_query_key_value_bias2: R.Tensor((3072,), dtype="float16") = packed_params[376] alloc504: R.Tensor((1, 1, 3072), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 3072]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10(gpt_neox_layers_23_attention_query_key_value_q_weight2, gpt_neox_layers_23_attention_query_key_value_q_scale2, alloc503, gpt_neox_layers_23_attention_query_key_value_bias2, alloc504) R.vm.kill_object(alloc503) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_weight2) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_scale2) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_bias2) lv430: R.Tensor((1, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc504, R.shape([1, 48, 64]), sinfo_args=(R.Tensor((1, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc504) alloc505: R.Tensor((1, 16, 64), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 16, 64]), R.dtype("float16")) _505: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), lv430, alloc505) R.vm.kill_object(lv430) lv431: R.Tensor((1, 1, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc505, R.shape([1, 1, 1024]), sinfo_args=(R.Tensor((1, 1, 1024), dtype="float16"),)) R.vm.kill_object(alloc505) gpt_neox_layers_23_post_attention_layernorm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[372] gpt_neox_layers_23_post_attention_layernorm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[373] alloc506: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm2(alloc502, gpt_neox_layers_23_post_attention_layernorm_weight2, gpt_neox_layers_23_post_attention_layernorm_bias2, alloc506) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_weight2) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_bias2) gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight2: R.Tensor((4096, 128), dtype="uint32") = packed_params[380] gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale2: R.Tensor((4096, 32), dtype="float16") = packed_params[381] gpt_neox_layers_23_mlp_dense_h_to_4h_bias2: R.Tensor((4096,), dtype="float32") = packed_params[382] alloc507: R.Tensor((1, 1, 4096), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 4096]), R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale2, alloc506, gpt_neox_layers_23_mlp_dense_h_to_4h_bias2, alloc507) R.vm.kill_object(alloc506) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight2) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale2) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_bias2) gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight2: R.Tensor((1024, 512), dtype="uint32") = packed_params[383] gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale2: R.Tensor((1024, 128), dtype="float16") = packed_params[384] gpt_neox_layers_23_mlp_dense_4h_to_h_bias2: R.Tensor((1024,), dtype="float32") = packed_params[385] alloc508: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage13, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) R.vm.kill_object(storage13) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale2, alloc507, gpt_neox_layers_23_mlp_dense_4h_to_h_bias2, alloc508) R.vm.kill_object(alloc507) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight2) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale2) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_bias2) gpt_neox_layers_23_attention_dense_q_weight2: R.Tensor((1024, 128), dtype="uint32") = packed_params[377] gpt_neox_layers_23_attention_dense_q_scale2: R.Tensor((1024, 32), dtype="float16") = packed_params[378] gpt_neox_layers_23_attention_dense_bias2: R.Tensor((1024,), dtype="float16") = packed_params[379] alloc509: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage15, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) R.vm.kill_object(storage15) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14(gpt_neox_layers_23_attention_dense_q_weight2, gpt_neox_layers_23_attention_dense_q_scale2, lv431, gpt_neox_layers_23_attention_dense_bias2, alloc508, alloc502, alloc509) R.vm.kill_object(alloc502) R.vm.kill_object(lv431) R.vm.kill_object(alloc508) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_weight2) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_scale2) R.vm.kill_object(gpt_neox_layers_23_attention_dense_bias2) gpt_neox_final_layer_norm_weight2: R.Tensor((1024,), dtype="float16") = packed_params[386] gpt_neox_final_layer_norm_bias2: R.Tensor((1024,), dtype="float16") = packed_params[387] alloc510: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) R.vm.kill_object(storage12) cls.gpt_neox_q4f16_1_layer_norm2(alloc509, gpt_neox_final_layer_norm_weight2, gpt_neox_final_layer_norm_bias2, alloc510) R.vm.kill_object(alloc509) R.vm.kill_object(gpt_neox_final_layer_norm_weight2) R.vm.kill_object(gpt_neox_final_layer_norm_bias2) embed_out_q_weight2: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] embed_out_q_scale2: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] gv553: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage16: R.Object = R.vm.alloc_storage(gv553, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv554: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), sinfo_args=(R.Shape(ndim=3),)) alloc511: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv554, R.dtype("float32")) R.vm.kill_object(storage16) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8(embed_out_q_weight2, embed_out_q_scale2, alloc510, alloc511) R.vm.kill_object(alloc510) R.vm.kill_object(embed_out_q_weight2) R.vm.kill_object(embed_out_q_scale2) gv2: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc511, paged_kv_cache R.vm.kill_object(alloc511) gv555: R.Tensor(dtype="float32", ndim=3) = gv2[0] R.call_packed("vm.builtin.match_shape", gv555, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(0), R.str("ErrorContext(fn=decode, loc=return, annotation=R.Tuple(R.Tensor((1, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv2 @R.function def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"))) -> R.Tensor(("seq_len", 1024), dtype="float16"): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(2),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_ids, R.prim_value(1), R.dtype("int32"), R.str("ErrorContext(fn=embed, loc=param[0], param=input_ids, annotation=R.Tensor((seq_len,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(390), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv544: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv544, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv545: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv545, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv546: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] R.call_packed("vm.builtin.check_tensor_info", gv546, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv547: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] R.call_packed("vm.builtin.check_tensor_info", gv547, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_ids, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(0), R.str("ErrorContext(fn=embed, loc=param[0], param=input_ids, annotation=R.Tensor((seq_len,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv544, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv545, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv546, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv547, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gpt_neox_embed_in_q_weight: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] gpt_neox_embed_in_q_scale: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] storage11: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv548: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=2),)) alloc341: R.Tensor(dtype="float16", ndim=2) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv548, R.dtype("float16")) R.vm.kill_object(storage11) cls.gpt_neox_q4f16_1_fused_dequantize_take1(gpt_neox_embed_in_q_weight, gpt_neox_embed_in_q_scale, input_ids, alloc341) R.vm.kill_object(gpt_neox_embed_in_q_weight) R.vm.kill_object(gpt_neox_embed_in_q_scale) R.call_packed("vm.builtin.match_shape", alloc341, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(1024), R.str("ErrorContext(fn=embed, loc=return, annotation=R.Tensor((seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) return alloc341 @R.function def prefill(input_embed: R.Tensor((1, "seq_len", 1024), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((3072, 128), dtype="uint32"), R.Tensor((3072, 32), dtype="float16"), R.Tensor((3072,), dtype="float16"), R.Tensor((1024, 128), dtype="uint32"), R.Tensor((1024, 32), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor((4096, 128), dtype="uint32"), R.Tensor((4096, 32), dtype="float16"), R.Tensor((4096,), dtype="float32"), R.Tensor((1024, 512), dtype="uint32"), R.Tensor((1024, 128), dtype="float16"), R.Tensor((1024,), dtype="float32"), R.Tensor((1024,), dtype="float16"), R.Tensor((1024,), dtype="float16"), R.Tensor(("vocab_size", 128), dtype="uint32"), R.Tensor(("vocab_size", 32), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embed, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[0], param=input_embed, annotation=R.Tensor((1, seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(390), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv1: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[1] R.call_packed("vm.builtin.check_tensor_info", gv1, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv2: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] R.call_packed("vm.builtin.check_tensor_info", gv2, R.prim_value(2), R.dtype("uint32"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv3: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] R.call_packed("vm.builtin.check_tensor_info", gv3, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embed, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), R.str("ErrorContext(fn=prefill, loc=param[0], param=input_embed, annotation=R.Tensor((1, seq_len, 1024), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv1, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv2, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(128), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv3, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(32), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((3072, 128), dtype=\"uint32\"), R.Tensor((3072, 32), dtype=\"float16\"), R.Tensor((3072,), dtype=\"float16\"), R.Tensor((1024, 128), dtype=\"uint32\"), R.Tensor((1024, 32), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((4096, 128), dtype=\"uint32\"), R.Tensor((4096, 32), dtype=\"float16\"), R.Tensor((4096,), dtype=\"float32\"), R.Tensor((1024, 512), dtype=\"uint32\"), R.Tensor((1024, 128), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float32\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((1024,), dtype=\"float16\"), R.Tensor((vocab_size, 128), dtype=\"uint32\"), R.Tensor((vocab_size, 32), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func(shape_heap) gpt_neox_layers_0_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[3] storage: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv4: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv4, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embed, gpt_neox_layers_0_input_layernorm_weight1, gpt_neox_layers_0_input_layernorm_bias1, alloc) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_0_input_layernorm_bias1) gpt_neox_layers_0_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[8] storage1: R.Object = R.vm.alloc_storage(R.shape([12582912]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv5: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc1: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv5, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_0_attention_query_key_value_q_weight1, gpt_neox_layers_0_attention_query_key_value_q_scale1, alloc, gpt_neox_layers_0_attention_query_key_value_bias1, alloc1) R.vm.kill_object(alloc) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_0_attention_query_key_value_bias1) gv6: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc1, gv6, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc1) gv7: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape1: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape, gv7, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape) gv8: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc2: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv8, R.dtype("float16")) _2: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1, alloc2) R.vm.kill_object(reshape1) gv9: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape2: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc2, gv9, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc2) gv10: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape3: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape2, gv10, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape2) gpt_neox_layers_0_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[5] gv11: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc3: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv11, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight1, gpt_neox_layers_0_post_attention_layernorm_bias1, alloc3) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_0_post_attention_layernorm_bias1) gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[14] storage2: R.Object = R.vm.alloc_storage(R.shape([16777216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv12: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc4: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv12, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1, alloc3, gpt_neox_layers_0_mlp_dense_h_to_4h_bias1, alloc4) R.vm.kill_object(alloc3) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_h_to_4h_bias1) gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[17] gv13: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc5: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv13, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1, alloc4, gpt_neox_layers_0_mlp_dense_4h_to_h_bias1, alloc5) R.vm.kill_object(alloc4) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_0_mlp_dense_4h_to_h_bias1) gpt_neox_layers_0_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[11] gv14: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc6: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv14, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_0_attention_dense_q_weight1, gpt_neox_layers_0_attention_dense_q_scale1, reshape3, gpt_neox_layers_0_attention_dense_bias1, alloc5, input_embed, alloc6) R.vm.kill_object(reshape3) R.vm.kill_object(alloc5) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_0_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_0_attention_dense_bias1) gpt_neox_layers_1_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[19] gv15: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc7: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv15, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc6, gpt_neox_layers_1_input_layernorm_weight1, gpt_neox_layers_1_input_layernorm_bias1, alloc7) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_1_input_layernorm_bias1) gpt_neox_layers_1_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[24] gv16: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc8: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv16, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_1_attention_query_key_value_q_weight1, gpt_neox_layers_1_attention_query_key_value_q_scale1, alloc7, gpt_neox_layers_1_attention_query_key_value_bias1, alloc8) R.vm.kill_object(alloc7) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_1_attention_query_key_value_bias1) gv17: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape4: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc8, gv17, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc8) gv18: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape5: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape4, gv18, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape4) gv19: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc9: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv19, R.dtype("float16")) _9: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5, alloc9) R.vm.kill_object(reshape5) gv20: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape6: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc9, gv20, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc9) gv21: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape7: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape6, gv21, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape6) gpt_neox_layers_1_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[21] gv22: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc10: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv22, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc6, gpt_neox_layers_1_post_attention_layernorm_weight1, gpt_neox_layers_1_post_attention_layernorm_bias1, alloc10) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_1_post_attention_layernorm_bias1) gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[30] storage3: R.Object = R.vm.alloc_storage(R.shape([16777216]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv23: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc11: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv23, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1, alloc10, gpt_neox_layers_1_mlp_dense_h_to_4h_bias1, alloc11) R.vm.kill_object(alloc10) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_h_to_4h_bias1) gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[33] gv24: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc12: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv24, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1, alloc11, gpt_neox_layers_1_mlp_dense_4h_to_h_bias1, alloc12) R.vm.kill_object(alloc11) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_1_mlp_dense_4h_to_h_bias1) gpt_neox_layers_1_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[27] gv25: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc13: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv25, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_1_attention_dense_q_weight1, gpt_neox_layers_1_attention_dense_q_scale1, reshape7, gpt_neox_layers_1_attention_dense_bias1, alloc12, alloc6, alloc13) R.vm.kill_object(alloc6) R.vm.kill_object(reshape7) R.vm.kill_object(alloc12) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_1_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_1_attention_dense_bias1) gpt_neox_layers_2_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[35] gv26: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc14: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv26, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc13, gpt_neox_layers_2_input_layernorm_weight1, gpt_neox_layers_2_input_layernorm_bias1, alloc14) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_2_input_layernorm_bias1) gpt_neox_layers_2_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[40] gv27: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc15: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv27, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_2_attention_query_key_value_q_weight1, gpt_neox_layers_2_attention_query_key_value_q_scale1, alloc14, gpt_neox_layers_2_attention_query_key_value_bias1, alloc15) R.vm.kill_object(alloc14) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_2_attention_query_key_value_bias1) gv28: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape8: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc15, gv28, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc15) gv29: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape9: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape8, gv29, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape8) gv30: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc16: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv30, R.dtype("float16")) _16: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9, alloc16) R.vm.kill_object(reshape9) gv31: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape10: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc16, gv31, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc16) gv32: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape11: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape10, gv32, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape10) gpt_neox_layers_2_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[37] gv33: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc17: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv33, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc13, gpt_neox_layers_2_post_attention_layernorm_weight1, gpt_neox_layers_2_post_attention_layernorm_bias1, alloc17) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_2_post_attention_layernorm_bias1) gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[46] gv34: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc18: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv34, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1, alloc17, gpt_neox_layers_2_mlp_dense_h_to_4h_bias1, alloc18) R.vm.kill_object(alloc17) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_h_to_4h_bias1) gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[49] gv35: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc19: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv35, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1, alloc18, gpt_neox_layers_2_mlp_dense_4h_to_h_bias1, alloc19) R.vm.kill_object(alloc18) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_2_mlp_dense_4h_to_h_bias1) gpt_neox_layers_2_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[43] gv36: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc20: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv36, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_2_attention_dense_q_weight1, gpt_neox_layers_2_attention_dense_q_scale1, reshape11, gpt_neox_layers_2_attention_dense_bias1, alloc19, alloc13, alloc20) R.vm.kill_object(alloc13) R.vm.kill_object(reshape11) R.vm.kill_object(alloc19) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_2_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_2_attention_dense_bias1) gpt_neox_layers_3_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[51] gv37: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc21: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv37, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc20, gpt_neox_layers_3_input_layernorm_weight1, gpt_neox_layers_3_input_layernorm_bias1, alloc21) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_3_input_layernorm_bias1) gpt_neox_layers_3_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[56] gv38: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc22: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv38, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_3_attention_query_key_value_q_weight1, gpt_neox_layers_3_attention_query_key_value_q_scale1, alloc21, gpt_neox_layers_3_attention_query_key_value_bias1, alloc22) R.vm.kill_object(alloc21) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_3_attention_query_key_value_bias1) gv39: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape12: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc22, gv39, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc22) gv40: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape13: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape12, gv40, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape12) gv41: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc23: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv41, R.dtype("float16")) _23: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13, alloc23) R.vm.kill_object(reshape13) gv42: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape14: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc23, gv42, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc23) gv43: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape15: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape14, gv43, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape14) gpt_neox_layers_3_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[53] gv44: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc24: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv44, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc20, gpt_neox_layers_3_post_attention_layernorm_weight1, gpt_neox_layers_3_post_attention_layernorm_bias1, alloc24) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_3_post_attention_layernorm_bias1) gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[62] gv45: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc25: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv45, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1, alloc24, gpt_neox_layers_3_mlp_dense_h_to_4h_bias1, alloc25) R.vm.kill_object(alloc24) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_h_to_4h_bias1) gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[65] gv46: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc26: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv46, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1, alloc25, gpt_neox_layers_3_mlp_dense_4h_to_h_bias1, alloc26) R.vm.kill_object(alloc25) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_3_mlp_dense_4h_to_h_bias1) gpt_neox_layers_3_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[59] gv47: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc27: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv47, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_3_attention_dense_q_weight1, gpt_neox_layers_3_attention_dense_q_scale1, reshape15, gpt_neox_layers_3_attention_dense_bias1, alloc26, alloc20, alloc27) R.vm.kill_object(alloc20) R.vm.kill_object(reshape15) R.vm.kill_object(alloc26) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_3_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_3_attention_dense_bias1) gpt_neox_layers_4_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[67] gv48: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc28: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv48, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc27, gpt_neox_layers_4_input_layernorm_weight1, gpt_neox_layers_4_input_layernorm_bias1, alloc28) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_4_input_layernorm_bias1) gpt_neox_layers_4_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[72] gv49: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc29: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv49, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_4_attention_query_key_value_q_weight1, gpt_neox_layers_4_attention_query_key_value_q_scale1, alloc28, gpt_neox_layers_4_attention_query_key_value_bias1, alloc29) R.vm.kill_object(alloc28) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_4_attention_query_key_value_bias1) gv50: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape16: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc29, gv50, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc29) gv51: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape17: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape16, gv51, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape16) gv52: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc30: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv52, R.dtype("float16")) _30: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17, alloc30) R.vm.kill_object(reshape17) gv53: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape18: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc30, gv53, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc30) gv54: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape19: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape18, gv54, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape18) gpt_neox_layers_4_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[69] gv55: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc31: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv55, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc27, gpt_neox_layers_4_post_attention_layernorm_weight1, gpt_neox_layers_4_post_attention_layernorm_bias1, alloc31) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_4_post_attention_layernorm_bias1) gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[78] gv56: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc32: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv56, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1, alloc31, gpt_neox_layers_4_mlp_dense_h_to_4h_bias1, alloc32) R.vm.kill_object(alloc31) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_h_to_4h_bias1) gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[81] gv57: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc33: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv57, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1, alloc32, gpt_neox_layers_4_mlp_dense_4h_to_h_bias1, alloc33) R.vm.kill_object(alloc32) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_4_mlp_dense_4h_to_h_bias1) gpt_neox_layers_4_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[75] gv58: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc34: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv58, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_4_attention_dense_q_weight1, gpt_neox_layers_4_attention_dense_q_scale1, reshape19, gpt_neox_layers_4_attention_dense_bias1, alloc33, alloc27, alloc34) R.vm.kill_object(alloc27) R.vm.kill_object(reshape19) R.vm.kill_object(alloc33) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_4_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_4_attention_dense_bias1) gpt_neox_layers_5_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[83] gv59: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc35: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv59, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc34, gpt_neox_layers_5_input_layernorm_weight1, gpt_neox_layers_5_input_layernorm_bias1, alloc35) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_5_input_layernorm_bias1) gpt_neox_layers_5_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[88] gv60: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc36: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv60, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_5_attention_query_key_value_q_weight1, gpt_neox_layers_5_attention_query_key_value_q_scale1, alloc35, gpt_neox_layers_5_attention_query_key_value_bias1, alloc36) R.vm.kill_object(alloc35) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_5_attention_query_key_value_bias1) gv61: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape20: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc36, gv61, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc36) gv62: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape21: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape20, gv62, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape20) gv63: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc37: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv63, R.dtype("float16")) _37: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21, alloc37) R.vm.kill_object(reshape21) gv64: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape22: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc37, gv64, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc37) gv65: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape23: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape22, gv65, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape22) gpt_neox_layers_5_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[85] gv66: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc38: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv66, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc34, gpt_neox_layers_5_post_attention_layernorm_weight1, gpt_neox_layers_5_post_attention_layernorm_bias1, alloc38) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_5_post_attention_layernorm_bias1) gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[94] gv67: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc39: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv67, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1, alloc38, gpt_neox_layers_5_mlp_dense_h_to_4h_bias1, alloc39) R.vm.kill_object(alloc38) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_h_to_4h_bias1) gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[97] gv68: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc40: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv68, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1, alloc39, gpt_neox_layers_5_mlp_dense_4h_to_h_bias1, alloc40) R.vm.kill_object(alloc39) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_5_mlp_dense_4h_to_h_bias1) gpt_neox_layers_5_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[91] gv69: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc41: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv69, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_5_attention_dense_q_weight1, gpt_neox_layers_5_attention_dense_q_scale1, reshape23, gpt_neox_layers_5_attention_dense_bias1, alloc40, alloc34, alloc41) R.vm.kill_object(alloc34) R.vm.kill_object(reshape23) R.vm.kill_object(alloc40) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_5_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_5_attention_dense_bias1) gpt_neox_layers_6_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[98] gpt_neox_layers_6_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[99] gv70: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc42: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv70, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc41, gpt_neox_layers_6_input_layernorm_weight1, gpt_neox_layers_6_input_layernorm_bias1, alloc42) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_6_input_layernorm_bias1) gpt_neox_layers_6_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[102] gpt_neox_layers_6_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[103] gpt_neox_layers_6_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[104] gv71: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc43: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv71, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_6_attention_query_key_value_q_weight1, gpt_neox_layers_6_attention_query_key_value_q_scale1, alloc42, gpt_neox_layers_6_attention_query_key_value_bias1, alloc43) R.vm.kill_object(alloc42) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_6_attention_query_key_value_bias1) gv72: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape24: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc43, gv72, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc43) gv73: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape25: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape24, gv73, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape24) gv74: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc44: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv74, R.dtype("float16")) _44: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape25, alloc44) R.vm.kill_object(reshape25) gv75: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape26: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc44, gv75, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc44) gv76: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape27: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape26, gv76, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape26) gpt_neox_layers_6_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[100] gpt_neox_layers_6_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[101] gv77: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc45: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv77, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc41, gpt_neox_layers_6_post_attention_layernorm_weight1, gpt_neox_layers_6_post_attention_layernorm_bias1, alloc45) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_6_post_attention_layernorm_bias1) gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[108] gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[109] gpt_neox_layers_6_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[110] gv78: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc46: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv78, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale1, alloc45, gpt_neox_layers_6_mlp_dense_h_to_4h_bias1, alloc46) R.vm.kill_object(alloc45) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_h_to_4h_bias1) gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[111] gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[112] gpt_neox_layers_6_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[113] gv79: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc47: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv79, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale1, alloc46, gpt_neox_layers_6_mlp_dense_4h_to_h_bias1, alloc47) R.vm.kill_object(alloc46) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_6_mlp_dense_4h_to_h_bias1) gpt_neox_layers_6_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[105] gpt_neox_layers_6_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[106] gpt_neox_layers_6_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[107] gv80: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc48: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv80, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_6_attention_dense_q_weight1, gpt_neox_layers_6_attention_dense_q_scale1, reshape27, gpt_neox_layers_6_attention_dense_bias1, alloc47, alloc41, alloc48) R.vm.kill_object(alloc41) R.vm.kill_object(reshape27) R.vm.kill_object(alloc47) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_6_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_6_attention_dense_bias1) gpt_neox_layers_7_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[114] gpt_neox_layers_7_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[115] gv81: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc49: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv81, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc48, gpt_neox_layers_7_input_layernorm_weight1, gpt_neox_layers_7_input_layernorm_bias1, alloc49) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_7_input_layernorm_bias1) gpt_neox_layers_7_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[118] gpt_neox_layers_7_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[119] gpt_neox_layers_7_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[120] gv82: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc50: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv82, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_7_attention_query_key_value_q_weight1, gpt_neox_layers_7_attention_query_key_value_q_scale1, alloc49, gpt_neox_layers_7_attention_query_key_value_bias1, alloc50) R.vm.kill_object(alloc49) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_7_attention_query_key_value_bias1) gv83: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape28: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc50, gv83, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc50) gv84: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape29: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape28, gv84, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape28) gv85: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc51: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv85, R.dtype("float16")) _51: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape29, alloc51) R.vm.kill_object(reshape29) gv86: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape30: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc51, gv86, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc51) gv87: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape31: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape30, gv87, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape30) gpt_neox_layers_7_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[116] gpt_neox_layers_7_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[117] gv88: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc52: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv88, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc48, gpt_neox_layers_7_post_attention_layernorm_weight1, gpt_neox_layers_7_post_attention_layernorm_bias1, alloc52) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_7_post_attention_layernorm_bias1) gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[124] gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[125] gpt_neox_layers_7_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[126] gv89: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc53: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv89, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale1, alloc52, gpt_neox_layers_7_mlp_dense_h_to_4h_bias1, alloc53) R.vm.kill_object(alloc52) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_h_to_4h_bias1) gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[127] gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[128] gpt_neox_layers_7_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[129] gv90: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc54: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv90, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale1, alloc53, gpt_neox_layers_7_mlp_dense_4h_to_h_bias1, alloc54) R.vm.kill_object(alloc53) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_7_mlp_dense_4h_to_h_bias1) gpt_neox_layers_7_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[121] gpt_neox_layers_7_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[122] gpt_neox_layers_7_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[123] gv91: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc55: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv91, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_7_attention_dense_q_weight1, gpt_neox_layers_7_attention_dense_q_scale1, reshape31, gpt_neox_layers_7_attention_dense_bias1, alloc54, alloc48, alloc55) R.vm.kill_object(alloc48) R.vm.kill_object(reshape31) R.vm.kill_object(alloc54) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_7_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_7_attention_dense_bias1) gpt_neox_layers_8_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[130] gpt_neox_layers_8_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[131] gv92: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc56: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv92, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc55, gpt_neox_layers_8_input_layernorm_weight1, gpt_neox_layers_8_input_layernorm_bias1, alloc56) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_8_input_layernorm_bias1) gpt_neox_layers_8_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[134] gpt_neox_layers_8_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[135] gpt_neox_layers_8_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[136] gv93: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc57: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv93, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_8_attention_query_key_value_q_weight1, gpt_neox_layers_8_attention_query_key_value_q_scale1, alloc56, gpt_neox_layers_8_attention_query_key_value_bias1, alloc57) R.vm.kill_object(alloc56) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_8_attention_query_key_value_bias1) gv94: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape32: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc57, gv94, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc57) gv95: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape33: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape32, gv95, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape32) gv96: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc58: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv96, R.dtype("float16")) _58: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape33, alloc58) R.vm.kill_object(reshape33) gv97: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape34: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc58, gv97, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc58) gv98: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape35: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape34, gv98, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape34) gpt_neox_layers_8_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[132] gpt_neox_layers_8_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[133] gv99: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc59: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv99, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc55, gpt_neox_layers_8_post_attention_layernorm_weight1, gpt_neox_layers_8_post_attention_layernorm_bias1, alloc59) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_8_post_attention_layernorm_bias1) gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[140] gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[141] gpt_neox_layers_8_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[142] gv100: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc60: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv100, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale1, alloc59, gpt_neox_layers_8_mlp_dense_h_to_4h_bias1, alloc60) R.vm.kill_object(alloc59) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_h_to_4h_bias1) gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[143] gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[144] gpt_neox_layers_8_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[145] gv101: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc61: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv101, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale1, alloc60, gpt_neox_layers_8_mlp_dense_4h_to_h_bias1, alloc61) R.vm.kill_object(alloc60) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_8_mlp_dense_4h_to_h_bias1) gpt_neox_layers_8_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[137] gpt_neox_layers_8_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[138] gpt_neox_layers_8_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[139] gv102: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc62: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv102, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_8_attention_dense_q_weight1, gpt_neox_layers_8_attention_dense_q_scale1, reshape35, gpt_neox_layers_8_attention_dense_bias1, alloc61, alloc55, alloc62) R.vm.kill_object(alloc55) R.vm.kill_object(reshape35) R.vm.kill_object(alloc61) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_8_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_8_attention_dense_bias1) gpt_neox_layers_9_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[146] gpt_neox_layers_9_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[147] gv103: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc63: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv103, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc62, gpt_neox_layers_9_input_layernorm_weight1, gpt_neox_layers_9_input_layernorm_bias1, alloc63) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_9_input_layernorm_bias1) gpt_neox_layers_9_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[150] gpt_neox_layers_9_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[151] gpt_neox_layers_9_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[152] gv104: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc64: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv104, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_9_attention_query_key_value_q_weight1, gpt_neox_layers_9_attention_query_key_value_q_scale1, alloc63, gpt_neox_layers_9_attention_query_key_value_bias1, alloc64) R.vm.kill_object(alloc63) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_9_attention_query_key_value_bias1) gv105: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape36: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc64, gv105, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc64) gv106: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape37: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape36, gv106, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape36) gv107: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc65: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv107, R.dtype("float16")) _65: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape37, alloc65) R.vm.kill_object(reshape37) gv108: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape38: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc65, gv108, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc65) gv109: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape39: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape38, gv109, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape38) gpt_neox_layers_9_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[148] gpt_neox_layers_9_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[149] gv110: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc66: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv110, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc62, gpt_neox_layers_9_post_attention_layernorm_weight1, gpt_neox_layers_9_post_attention_layernorm_bias1, alloc66) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_9_post_attention_layernorm_bias1) gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[156] gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[157] gpt_neox_layers_9_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[158] gv111: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc67: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv111, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale1, alloc66, gpt_neox_layers_9_mlp_dense_h_to_4h_bias1, alloc67) R.vm.kill_object(alloc66) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_h_to_4h_bias1) gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[159] gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[160] gpt_neox_layers_9_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[161] gv112: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc68: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv112, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale1, alloc67, gpt_neox_layers_9_mlp_dense_4h_to_h_bias1, alloc68) R.vm.kill_object(alloc67) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_9_mlp_dense_4h_to_h_bias1) gpt_neox_layers_9_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[153] gpt_neox_layers_9_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[154] gpt_neox_layers_9_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[155] gv113: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc69: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv113, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_9_attention_dense_q_weight1, gpt_neox_layers_9_attention_dense_q_scale1, reshape39, gpt_neox_layers_9_attention_dense_bias1, alloc68, alloc62, alloc69) R.vm.kill_object(alloc62) R.vm.kill_object(reshape39) R.vm.kill_object(alloc68) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_9_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_9_attention_dense_bias1) gpt_neox_layers_10_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[162] gpt_neox_layers_10_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[163] gv114: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc70: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv114, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc69, gpt_neox_layers_10_input_layernorm_weight1, gpt_neox_layers_10_input_layernorm_bias1, alloc70) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_10_input_layernorm_bias1) gpt_neox_layers_10_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[166] gpt_neox_layers_10_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[167] gpt_neox_layers_10_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[168] gv115: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc71: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv115, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_10_attention_query_key_value_q_weight1, gpt_neox_layers_10_attention_query_key_value_q_scale1, alloc70, gpt_neox_layers_10_attention_query_key_value_bias1, alloc71) R.vm.kill_object(alloc70) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_10_attention_query_key_value_bias1) gv116: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape40: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc71, gv116, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc71) gv117: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape41: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape40, gv117, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape40) gv118: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc72: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv118, R.dtype("float16")) _72: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape41, alloc72) R.vm.kill_object(reshape41) gv119: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape42: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc72, gv119, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc72) gv120: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape43: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape42, gv120, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape42) gpt_neox_layers_10_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[164] gpt_neox_layers_10_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[165] gv121: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc73: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv121, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc69, gpt_neox_layers_10_post_attention_layernorm_weight1, gpt_neox_layers_10_post_attention_layernorm_bias1, alloc73) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_10_post_attention_layernorm_bias1) gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[172] gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[173] gpt_neox_layers_10_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[174] gv122: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc74: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv122, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale1, alloc73, gpt_neox_layers_10_mlp_dense_h_to_4h_bias1, alloc74) R.vm.kill_object(alloc73) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_h_to_4h_bias1) gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[175] gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[176] gpt_neox_layers_10_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[177] gv123: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc75: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv123, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale1, alloc74, gpt_neox_layers_10_mlp_dense_4h_to_h_bias1, alloc75) R.vm.kill_object(alloc74) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_10_mlp_dense_4h_to_h_bias1) gpt_neox_layers_10_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[169] gpt_neox_layers_10_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[170] gpt_neox_layers_10_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[171] gv124: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc76: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv124, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_10_attention_dense_q_weight1, gpt_neox_layers_10_attention_dense_q_scale1, reshape43, gpt_neox_layers_10_attention_dense_bias1, alloc75, alloc69, alloc76) R.vm.kill_object(alloc69) R.vm.kill_object(reshape43) R.vm.kill_object(alloc75) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_10_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_10_attention_dense_bias1) gpt_neox_layers_11_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[178] gpt_neox_layers_11_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[179] gv125: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc77: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv125, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc76, gpt_neox_layers_11_input_layernorm_weight1, gpt_neox_layers_11_input_layernorm_bias1, alloc77) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_11_input_layernorm_bias1) gpt_neox_layers_11_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[182] gpt_neox_layers_11_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[183] gpt_neox_layers_11_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[184] gv126: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc78: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv126, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_11_attention_query_key_value_q_weight1, gpt_neox_layers_11_attention_query_key_value_q_scale1, alloc77, gpt_neox_layers_11_attention_query_key_value_bias1, alloc78) R.vm.kill_object(alloc77) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_11_attention_query_key_value_bias1) gv127: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape44: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc78, gv127, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc78) gv128: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape45: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape44, gv128, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape44) gv129: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc79: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv129, R.dtype("float16")) _79: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape45, alloc79) R.vm.kill_object(reshape45) gv130: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape46: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc79, gv130, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc79) gv131: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape47: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape46, gv131, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape46) gpt_neox_layers_11_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[180] gpt_neox_layers_11_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[181] gv132: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc80: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv132, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc76, gpt_neox_layers_11_post_attention_layernorm_weight1, gpt_neox_layers_11_post_attention_layernorm_bias1, alloc80) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_11_post_attention_layernorm_bias1) gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[188] gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[189] gpt_neox_layers_11_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[190] gv133: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc81: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv133, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale1, alloc80, gpt_neox_layers_11_mlp_dense_h_to_4h_bias1, alloc81) R.vm.kill_object(alloc80) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_h_to_4h_bias1) gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[191] gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[192] gpt_neox_layers_11_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[193] gv134: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc82: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv134, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale1, alloc81, gpt_neox_layers_11_mlp_dense_4h_to_h_bias1, alloc82) R.vm.kill_object(alloc81) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_11_mlp_dense_4h_to_h_bias1) gpt_neox_layers_11_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[185] gpt_neox_layers_11_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[186] gpt_neox_layers_11_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[187] gv135: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc83: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv135, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_11_attention_dense_q_weight1, gpt_neox_layers_11_attention_dense_q_scale1, reshape47, gpt_neox_layers_11_attention_dense_bias1, alloc82, alloc76, alloc83) R.vm.kill_object(alloc76) R.vm.kill_object(reshape47) R.vm.kill_object(alloc82) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_11_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_11_attention_dense_bias1) gpt_neox_layers_12_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[194] gpt_neox_layers_12_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[195] gv136: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc84: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv136, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc83, gpt_neox_layers_12_input_layernorm_weight1, gpt_neox_layers_12_input_layernorm_bias1, alloc84) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_12_input_layernorm_bias1) gpt_neox_layers_12_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[198] gpt_neox_layers_12_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[199] gpt_neox_layers_12_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[200] gv137: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc85: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv137, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_12_attention_query_key_value_q_weight1, gpt_neox_layers_12_attention_query_key_value_q_scale1, alloc84, gpt_neox_layers_12_attention_query_key_value_bias1, alloc85) R.vm.kill_object(alloc84) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_12_attention_query_key_value_bias1) gv138: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape48: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc85, gv138, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc85) gv139: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape49: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape48, gv139, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape48) gv140: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc86: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv140, R.dtype("float16")) _86: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape49, alloc86) R.vm.kill_object(reshape49) gv141: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape50: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc86, gv141, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc86) gv142: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape51: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape50, gv142, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape50) gpt_neox_layers_12_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[196] gpt_neox_layers_12_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[197] gv143: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc87: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv143, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc83, gpt_neox_layers_12_post_attention_layernorm_weight1, gpt_neox_layers_12_post_attention_layernorm_bias1, alloc87) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_12_post_attention_layernorm_bias1) gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[204] gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[205] gpt_neox_layers_12_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[206] gv144: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc88: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv144, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale1, alloc87, gpt_neox_layers_12_mlp_dense_h_to_4h_bias1, alloc88) R.vm.kill_object(alloc87) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_h_to_4h_bias1) gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[207] gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[208] gpt_neox_layers_12_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[209] gv145: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc89: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv145, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale1, alloc88, gpt_neox_layers_12_mlp_dense_4h_to_h_bias1, alloc89) R.vm.kill_object(alloc88) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_12_mlp_dense_4h_to_h_bias1) gpt_neox_layers_12_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[201] gpt_neox_layers_12_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[202] gpt_neox_layers_12_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[203] gv146: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc90: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv146, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_12_attention_dense_q_weight1, gpt_neox_layers_12_attention_dense_q_scale1, reshape51, gpt_neox_layers_12_attention_dense_bias1, alloc89, alloc83, alloc90) R.vm.kill_object(alloc83) R.vm.kill_object(reshape51) R.vm.kill_object(alloc89) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_12_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_12_attention_dense_bias1) gpt_neox_layers_13_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[210] gpt_neox_layers_13_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[211] gv147: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc91: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv147, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc90, gpt_neox_layers_13_input_layernorm_weight1, gpt_neox_layers_13_input_layernorm_bias1, alloc91) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_13_input_layernorm_bias1) gpt_neox_layers_13_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[214] gpt_neox_layers_13_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[215] gpt_neox_layers_13_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[216] gv148: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc92: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv148, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_13_attention_query_key_value_q_weight1, gpt_neox_layers_13_attention_query_key_value_q_scale1, alloc91, gpt_neox_layers_13_attention_query_key_value_bias1, alloc92) R.vm.kill_object(alloc91) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_13_attention_query_key_value_bias1) gv149: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape52: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc92, gv149, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc92) gv150: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape53: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape52, gv150, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape52) gv151: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc93: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv151, R.dtype("float16")) _93: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape53, alloc93) R.vm.kill_object(reshape53) gv152: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape54: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc93, gv152, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc93) gv153: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape55: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape54, gv153, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape54) gpt_neox_layers_13_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[212] gpt_neox_layers_13_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[213] gv154: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc94: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv154, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc90, gpt_neox_layers_13_post_attention_layernorm_weight1, gpt_neox_layers_13_post_attention_layernorm_bias1, alloc94) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_13_post_attention_layernorm_bias1) gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[220] gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[221] gpt_neox_layers_13_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[222] gv155: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc95: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv155, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale1, alloc94, gpt_neox_layers_13_mlp_dense_h_to_4h_bias1, alloc95) R.vm.kill_object(alloc94) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_h_to_4h_bias1) gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[223] gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[224] gpt_neox_layers_13_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[225] gv156: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc96: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv156, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale1, alloc95, gpt_neox_layers_13_mlp_dense_4h_to_h_bias1, alloc96) R.vm.kill_object(alloc95) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_13_mlp_dense_4h_to_h_bias1) gpt_neox_layers_13_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[217] gpt_neox_layers_13_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[218] gpt_neox_layers_13_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[219] gv157: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc97: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv157, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_13_attention_dense_q_weight1, gpt_neox_layers_13_attention_dense_q_scale1, reshape55, gpt_neox_layers_13_attention_dense_bias1, alloc96, alloc90, alloc97) R.vm.kill_object(alloc90) R.vm.kill_object(reshape55) R.vm.kill_object(alloc96) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_13_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_13_attention_dense_bias1) gpt_neox_layers_14_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[226] gpt_neox_layers_14_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[227] gv158: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc98: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv158, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc97, gpt_neox_layers_14_input_layernorm_weight1, gpt_neox_layers_14_input_layernorm_bias1, alloc98) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_14_input_layernorm_bias1) gpt_neox_layers_14_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[230] gpt_neox_layers_14_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[231] gpt_neox_layers_14_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[232] gv159: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc99: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv159, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_14_attention_query_key_value_q_weight1, gpt_neox_layers_14_attention_query_key_value_q_scale1, alloc98, gpt_neox_layers_14_attention_query_key_value_bias1, alloc99) R.vm.kill_object(alloc98) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_14_attention_query_key_value_bias1) gv160: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape56: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc99, gv160, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc99) gv161: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape57: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape56, gv161, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape56) gv162: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc100: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv162, R.dtype("float16")) _100: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape57, alloc100) R.vm.kill_object(reshape57) gv163: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape58: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc100, gv163, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc100) gv164: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape59: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape58, gv164, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape58) gpt_neox_layers_14_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[228] gpt_neox_layers_14_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[229] gv165: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc101: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv165, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc97, gpt_neox_layers_14_post_attention_layernorm_weight1, gpt_neox_layers_14_post_attention_layernorm_bias1, alloc101) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_14_post_attention_layernorm_bias1) gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[236] gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[237] gpt_neox_layers_14_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[238] gv166: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc102: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv166, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale1, alloc101, gpt_neox_layers_14_mlp_dense_h_to_4h_bias1, alloc102) R.vm.kill_object(alloc101) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_h_to_4h_bias1) gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[239] gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[240] gpt_neox_layers_14_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[241] gv167: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc103: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv167, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale1, alloc102, gpt_neox_layers_14_mlp_dense_4h_to_h_bias1, alloc103) R.vm.kill_object(alloc102) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_14_mlp_dense_4h_to_h_bias1) gpt_neox_layers_14_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[233] gpt_neox_layers_14_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[234] gpt_neox_layers_14_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[235] gv168: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc104: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv168, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_14_attention_dense_q_weight1, gpt_neox_layers_14_attention_dense_q_scale1, reshape59, gpt_neox_layers_14_attention_dense_bias1, alloc103, alloc97, alloc104) R.vm.kill_object(alloc97) R.vm.kill_object(reshape59) R.vm.kill_object(alloc103) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_14_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_14_attention_dense_bias1) gpt_neox_layers_15_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[242] gpt_neox_layers_15_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[243] gv169: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc105: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv169, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc104, gpt_neox_layers_15_input_layernorm_weight1, gpt_neox_layers_15_input_layernorm_bias1, alloc105) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_15_input_layernorm_bias1) gpt_neox_layers_15_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[246] gpt_neox_layers_15_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[247] gpt_neox_layers_15_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[248] gv170: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc106: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv170, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_15_attention_query_key_value_q_weight1, gpt_neox_layers_15_attention_query_key_value_q_scale1, alloc105, gpt_neox_layers_15_attention_query_key_value_bias1, alloc106) R.vm.kill_object(alloc105) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_15_attention_query_key_value_bias1) gv171: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape60: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc106, gv171, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc106) gv172: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape61: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape60, gv172, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape60) gv173: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc107: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv173, R.dtype("float16")) _107: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape61, alloc107) R.vm.kill_object(reshape61) gv174: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape62: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc107, gv174, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc107) gv175: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape63: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape62, gv175, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape62) gpt_neox_layers_15_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[244] gpt_neox_layers_15_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[245] gv176: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc108: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv176, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc104, gpt_neox_layers_15_post_attention_layernorm_weight1, gpt_neox_layers_15_post_attention_layernorm_bias1, alloc108) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_15_post_attention_layernorm_bias1) gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[252] gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[253] gpt_neox_layers_15_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[254] gv177: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc109: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv177, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale1, alloc108, gpt_neox_layers_15_mlp_dense_h_to_4h_bias1, alloc109) R.vm.kill_object(alloc108) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_h_to_4h_bias1) gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[255] gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[256] gpt_neox_layers_15_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[257] gv178: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc110: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv178, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale1, alloc109, gpt_neox_layers_15_mlp_dense_4h_to_h_bias1, alloc110) R.vm.kill_object(alloc109) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_15_mlp_dense_4h_to_h_bias1) gpt_neox_layers_15_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[249] gpt_neox_layers_15_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[250] gpt_neox_layers_15_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[251] gv179: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc111: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv179, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_15_attention_dense_q_weight1, gpt_neox_layers_15_attention_dense_q_scale1, reshape63, gpt_neox_layers_15_attention_dense_bias1, alloc110, alloc104, alloc111) R.vm.kill_object(alloc104) R.vm.kill_object(reshape63) R.vm.kill_object(alloc110) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_15_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_15_attention_dense_bias1) gpt_neox_layers_16_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[258] gpt_neox_layers_16_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[259] gv180: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc112: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv180, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc111, gpt_neox_layers_16_input_layernorm_weight1, gpt_neox_layers_16_input_layernorm_bias1, alloc112) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_16_input_layernorm_bias1) gpt_neox_layers_16_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[262] gpt_neox_layers_16_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[263] gpt_neox_layers_16_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[264] gv181: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc113: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv181, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_16_attention_query_key_value_q_weight1, gpt_neox_layers_16_attention_query_key_value_q_scale1, alloc112, gpt_neox_layers_16_attention_query_key_value_bias1, alloc113) R.vm.kill_object(alloc112) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_16_attention_query_key_value_bias1) gv182: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape64: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc113, gv182, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc113) gv183: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape65: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape64, gv183, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape64) gv184: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc114: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv184, R.dtype("float16")) _114: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape65, alloc114) R.vm.kill_object(reshape65) gv185: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape66: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc114, gv185, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc114) gv186: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape67: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape66, gv186, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape66) gpt_neox_layers_16_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[260] gpt_neox_layers_16_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[261] gv187: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc115: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv187, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc111, gpt_neox_layers_16_post_attention_layernorm_weight1, gpt_neox_layers_16_post_attention_layernorm_bias1, alloc115) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_16_post_attention_layernorm_bias1) gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[268] gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[269] gpt_neox_layers_16_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[270] gv188: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc116: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv188, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale1, alloc115, gpt_neox_layers_16_mlp_dense_h_to_4h_bias1, alloc116) R.vm.kill_object(alloc115) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_h_to_4h_bias1) gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[271] gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[272] gpt_neox_layers_16_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[273] gv189: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc117: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv189, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale1, alloc116, gpt_neox_layers_16_mlp_dense_4h_to_h_bias1, alloc117) R.vm.kill_object(alloc116) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_16_mlp_dense_4h_to_h_bias1) gpt_neox_layers_16_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[265] gpt_neox_layers_16_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[266] gpt_neox_layers_16_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[267] gv190: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc118: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv190, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_16_attention_dense_q_weight1, gpt_neox_layers_16_attention_dense_q_scale1, reshape67, gpt_neox_layers_16_attention_dense_bias1, alloc117, alloc111, alloc118) R.vm.kill_object(alloc111) R.vm.kill_object(reshape67) R.vm.kill_object(alloc117) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_16_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_16_attention_dense_bias1) gpt_neox_layers_17_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[274] gpt_neox_layers_17_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[275] gv191: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc119: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv191, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc118, gpt_neox_layers_17_input_layernorm_weight1, gpt_neox_layers_17_input_layernorm_bias1, alloc119) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_17_input_layernorm_bias1) gpt_neox_layers_17_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[278] gpt_neox_layers_17_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[279] gpt_neox_layers_17_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[280] gv192: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc120: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv192, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_17_attention_query_key_value_q_weight1, gpt_neox_layers_17_attention_query_key_value_q_scale1, alloc119, gpt_neox_layers_17_attention_query_key_value_bias1, alloc120) R.vm.kill_object(alloc119) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_17_attention_query_key_value_bias1) gv193: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape68: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc120, gv193, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc120) gv194: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape69: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape68, gv194, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape68) gv195: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc121: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv195, R.dtype("float16")) _121: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape69, alloc121) R.vm.kill_object(reshape69) gv196: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape70: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc121, gv196, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc121) gv197: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape71: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape70, gv197, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape70) gpt_neox_layers_17_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[276] gpt_neox_layers_17_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[277] gv198: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc122: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv198, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc118, gpt_neox_layers_17_post_attention_layernorm_weight1, gpt_neox_layers_17_post_attention_layernorm_bias1, alloc122) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_17_post_attention_layernorm_bias1) gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[284] gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[285] gpt_neox_layers_17_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[286] gv199: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc123: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv199, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale1, alloc122, gpt_neox_layers_17_mlp_dense_h_to_4h_bias1, alloc123) R.vm.kill_object(alloc122) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_h_to_4h_bias1) gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[287] gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[288] gpt_neox_layers_17_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[289] gv200: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc124: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv200, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale1, alloc123, gpt_neox_layers_17_mlp_dense_4h_to_h_bias1, alloc124) R.vm.kill_object(alloc123) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_17_mlp_dense_4h_to_h_bias1) gpt_neox_layers_17_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[281] gpt_neox_layers_17_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[282] gpt_neox_layers_17_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[283] gv201: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc125: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv201, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_17_attention_dense_q_weight1, gpt_neox_layers_17_attention_dense_q_scale1, reshape71, gpt_neox_layers_17_attention_dense_bias1, alloc124, alloc118, alloc125) R.vm.kill_object(alloc118) R.vm.kill_object(reshape71) R.vm.kill_object(alloc124) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_17_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_17_attention_dense_bias1) gpt_neox_layers_18_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[290] gpt_neox_layers_18_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[291] gv202: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc126: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv202, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc125, gpt_neox_layers_18_input_layernorm_weight1, gpt_neox_layers_18_input_layernorm_bias1, alloc126) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_18_input_layernorm_bias1) gpt_neox_layers_18_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[294] gpt_neox_layers_18_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[295] gpt_neox_layers_18_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[296] gv203: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc127: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv203, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_18_attention_query_key_value_q_weight1, gpt_neox_layers_18_attention_query_key_value_q_scale1, alloc126, gpt_neox_layers_18_attention_query_key_value_bias1, alloc127) R.vm.kill_object(alloc126) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_18_attention_query_key_value_bias1) gv204: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape72: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc127, gv204, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc127) gv205: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape73: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape72, gv205, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape72) gv206: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc128: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv206, R.dtype("float16")) _128: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape73, alloc128) R.vm.kill_object(reshape73) gv207: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape74: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc128, gv207, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc128) gv208: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape75: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape74, gv208, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape74) gpt_neox_layers_18_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[292] gpt_neox_layers_18_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[293] gv209: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc129: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv209, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc125, gpt_neox_layers_18_post_attention_layernorm_weight1, gpt_neox_layers_18_post_attention_layernorm_bias1, alloc129) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_18_post_attention_layernorm_bias1) gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[300] gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[301] gpt_neox_layers_18_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[302] gv210: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc130: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv210, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale1, alloc129, gpt_neox_layers_18_mlp_dense_h_to_4h_bias1, alloc130) R.vm.kill_object(alloc129) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_h_to_4h_bias1) gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[303] gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[304] gpt_neox_layers_18_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[305] gv211: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc131: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv211, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale1, alloc130, gpt_neox_layers_18_mlp_dense_4h_to_h_bias1, alloc131) R.vm.kill_object(alloc130) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_18_mlp_dense_4h_to_h_bias1) gpt_neox_layers_18_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[297] gpt_neox_layers_18_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[298] gpt_neox_layers_18_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[299] gv212: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc132: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv212, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_18_attention_dense_q_weight1, gpt_neox_layers_18_attention_dense_q_scale1, reshape75, gpt_neox_layers_18_attention_dense_bias1, alloc131, alloc125, alloc132) R.vm.kill_object(alloc125) R.vm.kill_object(reshape75) R.vm.kill_object(alloc131) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_18_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_18_attention_dense_bias1) gpt_neox_layers_19_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[306] gpt_neox_layers_19_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[307] gv213: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc133: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv213, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc132, gpt_neox_layers_19_input_layernorm_weight1, gpt_neox_layers_19_input_layernorm_bias1, alloc133) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_19_input_layernorm_bias1) gpt_neox_layers_19_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[310] gpt_neox_layers_19_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[311] gpt_neox_layers_19_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[312] gv214: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc134: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv214, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_19_attention_query_key_value_q_weight1, gpt_neox_layers_19_attention_query_key_value_q_scale1, alloc133, gpt_neox_layers_19_attention_query_key_value_bias1, alloc134) R.vm.kill_object(alloc133) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_19_attention_query_key_value_bias1) gv215: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape76: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc134, gv215, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc134) gv216: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape77: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape76, gv216, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape76) gv217: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc135: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv217, R.dtype("float16")) _135: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape77, alloc135) R.vm.kill_object(reshape77) gv218: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape78: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc135, gv218, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc135) gv219: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape79: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape78, gv219, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape78) gpt_neox_layers_19_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[308] gpt_neox_layers_19_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[309] gv220: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc136: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv220, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc132, gpt_neox_layers_19_post_attention_layernorm_weight1, gpt_neox_layers_19_post_attention_layernorm_bias1, alloc136) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_19_post_attention_layernorm_bias1) gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[316] gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[317] gpt_neox_layers_19_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[318] gv221: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc137: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv221, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale1, alloc136, gpt_neox_layers_19_mlp_dense_h_to_4h_bias1, alloc137) R.vm.kill_object(alloc136) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_h_to_4h_bias1) gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[319] gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[320] gpt_neox_layers_19_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[321] gv222: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc138: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv222, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale1, alloc137, gpt_neox_layers_19_mlp_dense_4h_to_h_bias1, alloc138) R.vm.kill_object(alloc137) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_19_mlp_dense_4h_to_h_bias1) gpt_neox_layers_19_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[313] gpt_neox_layers_19_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[314] gpt_neox_layers_19_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[315] gv223: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc139: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv223, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_19_attention_dense_q_weight1, gpt_neox_layers_19_attention_dense_q_scale1, reshape79, gpt_neox_layers_19_attention_dense_bias1, alloc138, alloc132, alloc139) R.vm.kill_object(alloc132) R.vm.kill_object(reshape79) R.vm.kill_object(alloc138) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_19_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_19_attention_dense_bias1) gpt_neox_layers_20_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[322] gpt_neox_layers_20_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[323] gv224: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc140: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv224, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc139, gpt_neox_layers_20_input_layernorm_weight1, gpt_neox_layers_20_input_layernorm_bias1, alloc140) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_20_input_layernorm_bias1) gpt_neox_layers_20_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[326] gpt_neox_layers_20_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[327] gpt_neox_layers_20_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[328] gv225: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc141: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv225, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_20_attention_query_key_value_q_weight1, gpt_neox_layers_20_attention_query_key_value_q_scale1, alloc140, gpt_neox_layers_20_attention_query_key_value_bias1, alloc141) R.vm.kill_object(alloc140) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_20_attention_query_key_value_bias1) gv226: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape80: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc141, gv226, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc141) gv227: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape81: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape80, gv227, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape80) gv228: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc142: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv228, R.dtype("float16")) _142: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape81, alloc142) R.vm.kill_object(reshape81) gv229: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape82: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc142, gv229, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc142) gv230: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape83: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape82, gv230, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape82) gpt_neox_layers_20_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[324] gpt_neox_layers_20_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[325] gv231: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc143: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv231, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc139, gpt_neox_layers_20_post_attention_layernorm_weight1, gpt_neox_layers_20_post_attention_layernorm_bias1, alloc143) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_20_post_attention_layernorm_bias1) gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[332] gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[333] gpt_neox_layers_20_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[334] gv232: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc144: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv232, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale1, alloc143, gpt_neox_layers_20_mlp_dense_h_to_4h_bias1, alloc144) R.vm.kill_object(alloc143) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_h_to_4h_bias1) gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[335] gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[336] gpt_neox_layers_20_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[337] gv233: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc145: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv233, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale1, alloc144, gpt_neox_layers_20_mlp_dense_4h_to_h_bias1, alloc145) R.vm.kill_object(alloc144) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_20_mlp_dense_4h_to_h_bias1) gpt_neox_layers_20_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[329] gpt_neox_layers_20_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[330] gpt_neox_layers_20_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[331] gv234: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc146: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv234, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_20_attention_dense_q_weight1, gpt_neox_layers_20_attention_dense_q_scale1, reshape83, gpt_neox_layers_20_attention_dense_bias1, alloc145, alloc139, alloc146) R.vm.kill_object(alloc139) R.vm.kill_object(reshape83) R.vm.kill_object(alloc145) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_20_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_20_attention_dense_bias1) gpt_neox_layers_21_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[338] gpt_neox_layers_21_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[339] gv235: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc147: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv235, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc146, gpt_neox_layers_21_input_layernorm_weight1, gpt_neox_layers_21_input_layernorm_bias1, alloc147) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_21_input_layernorm_bias1) gpt_neox_layers_21_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[342] gpt_neox_layers_21_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[343] gpt_neox_layers_21_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[344] gv236: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc148: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv236, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_21_attention_query_key_value_q_weight1, gpt_neox_layers_21_attention_query_key_value_q_scale1, alloc147, gpt_neox_layers_21_attention_query_key_value_bias1, alloc148) R.vm.kill_object(alloc147) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_21_attention_query_key_value_bias1) gv237: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape84: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc148, gv237, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc148) gv238: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape85: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape84, gv238, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape84) gv239: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc149: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv239, R.dtype("float16")) _149: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape85, alloc149) R.vm.kill_object(reshape85) gv240: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape86: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc149, gv240, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc149) gv241: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape87: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape86, gv241, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape86) gpt_neox_layers_21_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[340] gpt_neox_layers_21_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[341] gv242: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc150: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv242, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc146, gpt_neox_layers_21_post_attention_layernorm_weight1, gpt_neox_layers_21_post_attention_layernorm_bias1, alloc150) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_21_post_attention_layernorm_bias1) gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[348] gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[349] gpt_neox_layers_21_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[350] gv243: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc151: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv243, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale1, alloc150, gpt_neox_layers_21_mlp_dense_h_to_4h_bias1, alloc151) R.vm.kill_object(alloc150) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_h_to_4h_bias1) gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[351] gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[352] gpt_neox_layers_21_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[353] gv244: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc152: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv244, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale1, alloc151, gpt_neox_layers_21_mlp_dense_4h_to_h_bias1, alloc152) R.vm.kill_object(alloc151) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_21_mlp_dense_4h_to_h_bias1) gpt_neox_layers_21_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[345] gpt_neox_layers_21_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[346] gpt_neox_layers_21_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[347] gv245: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc153: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv245, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_21_attention_dense_q_weight1, gpt_neox_layers_21_attention_dense_q_scale1, reshape87, gpt_neox_layers_21_attention_dense_bias1, alloc152, alloc146, alloc153) R.vm.kill_object(alloc146) R.vm.kill_object(reshape87) R.vm.kill_object(alloc152) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_21_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_21_attention_dense_bias1) gpt_neox_layers_22_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[354] gpt_neox_layers_22_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[355] gv246: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc154: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv246, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc153, gpt_neox_layers_22_input_layernorm_weight1, gpt_neox_layers_22_input_layernorm_bias1, alloc154) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_22_input_layernorm_bias1) gpt_neox_layers_22_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[358] gpt_neox_layers_22_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[359] gpt_neox_layers_22_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[360] gv247: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc155: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv247, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_22_attention_query_key_value_q_weight1, gpt_neox_layers_22_attention_query_key_value_q_scale1, alloc154, gpt_neox_layers_22_attention_query_key_value_bias1, alloc155) R.vm.kill_object(alloc154) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_22_attention_query_key_value_bias1) gv248: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape88: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc155, gv248, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc155) gv249: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape89: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape88, gv249, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape88) gv250: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc156: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv250, R.dtype("float16")) _156: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape89, alloc156) R.vm.kill_object(reshape89) gv251: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape90: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc156, gv251, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc156) gv252: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape91: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape90, gv252, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape90) gpt_neox_layers_22_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[356] gpt_neox_layers_22_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[357] gv253: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc157: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv253, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc153, gpt_neox_layers_22_post_attention_layernorm_weight1, gpt_neox_layers_22_post_attention_layernorm_bias1, alloc157) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_22_post_attention_layernorm_bias1) gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[364] gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[365] gpt_neox_layers_22_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[366] gv254: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc158: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv254, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale1, alloc157, gpt_neox_layers_22_mlp_dense_h_to_4h_bias1, alloc158) R.vm.kill_object(alloc157) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_h_to_4h_bias1) gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[367] gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[368] gpt_neox_layers_22_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[369] gv255: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc159: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv255, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale1, alloc158, gpt_neox_layers_22_mlp_dense_4h_to_h_bias1, alloc159) R.vm.kill_object(alloc158) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_22_mlp_dense_4h_to_h_bias1) gpt_neox_layers_22_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[361] gpt_neox_layers_22_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[362] gpt_neox_layers_22_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[363] gv256: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc160: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv256, R.dtype("float16")) R.vm.kill_object(storage2) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_22_attention_dense_q_weight1, gpt_neox_layers_22_attention_dense_q_scale1, reshape91, gpt_neox_layers_22_attention_dense_bias1, alloc159, alloc153, alloc160) R.vm.kill_object(alloc153) R.vm.kill_object(reshape91) R.vm.kill_object(alloc159) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_22_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_22_attention_dense_bias1) gpt_neox_layers_23_input_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[370] gpt_neox_layers_23_input_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[371] gv257: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc161: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv257, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc160, gpt_neox_layers_23_input_layernorm_weight1, gpt_neox_layers_23_input_layernorm_bias1, alloc161) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_23_input_layernorm_bias1) gpt_neox_layers_23_attention_query_key_value_q_weight1: R.Tensor((3072, 128), dtype="uint32") = packed_params[374] gpt_neox_layers_23_attention_query_key_value_q_scale1: R.Tensor((3072, 32), dtype="float16") = packed_params[375] gpt_neox_layers_23_attention_query_key_value_bias1: R.Tensor((3072,), dtype="float16") = packed_params[376] gv258: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(3072), sinfo_args=(R.Shape(ndim=3),)) alloc162: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv258, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5(gpt_neox_layers_23_attention_query_key_value_q_weight1, gpt_neox_layers_23_attention_query_key_value_q_scale1, alloc161, gpt_neox_layers_23_attention_query_key_value_bias1, alloc162) R.vm.kill_object(alloc161) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_weight1) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_q_scale1) R.vm.kill_object(gpt_neox_layers_23_attention_query_key_value_bias1) gv259: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape92: R.Tensor((1, seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc162, gv259, sinfo_args=(R.Tensor((1, seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(alloc162) gv260: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(48), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) reshape93: R.Tensor((seq_len, 48, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape92, gv260, sinfo_args=(R.Tensor((seq_len, 48, 64), dtype="float16"),)) R.vm.kill_object(reshape92) gv261: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=3),)) alloc163: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv261, R.dtype("float16")) _163: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape93, alloc163) R.vm.kill_object(reshape93) gv262: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16), R.prim_value(0), R.prim_value(64), sinfo_args=(R.Shape(ndim=4),)) reshape94: R.Tensor((1, seq_len, 16, 64), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc163, gv262, sinfo_args=(R.Tensor((1, seq_len, 16, 64), dtype="float16"),)) R.vm.kill_object(alloc163) gv263: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) reshape95: R.Tensor((1, seq_len, 1024), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape94, gv263, sinfo_args=(R.Tensor((1, seq_len, 1024), dtype="float16"),)) R.vm.kill_object(reshape94) gpt_neox_layers_23_post_attention_layernorm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[372] gpt_neox_layers_23_post_attention_layernorm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[373] gv264: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc164: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv264, R.dtype("float16")) cls.gpt_neox_q4f16_1_layer_norm1(alloc160, gpt_neox_layers_23_post_attention_layernorm_weight1, gpt_neox_layers_23_post_attention_layernorm_bias1, alloc164) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_weight1) R.vm.kill_object(gpt_neox_layers_23_post_attention_layernorm_bias1) gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight1: R.Tensor((4096, 128), dtype="uint32") = packed_params[380] gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale1: R.Tensor((4096, 32), dtype="float16") = packed_params[381] gpt_neox_layers_23_mlp_dense_h_to_4h_bias1: R.Tensor((4096,), dtype="float32") = packed_params[382] gv265: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(4096), sinfo_args=(R.Shape(ndim=3),)) alloc165: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv265, R.dtype("float16")) cls.gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale1, alloc164, gpt_neox_layers_23_mlp_dense_h_to_4h_bias1, alloc165) R.vm.kill_object(alloc164) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_weight1) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_q_scale1) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_h_to_4h_bias1) gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight1: R.Tensor((1024, 512), dtype="uint32") = packed_params[383] gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale1: R.Tensor((1024, 128), dtype="float16") = packed_params[384] gpt_neox_layers_23_mlp_dense_4h_to_h_bias1: R.Tensor((1024,), dtype="float32") = packed_params[385] gv266: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc166: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv266, R.dtype("float16")) R.vm.kill_object(storage1) cls.gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale1, alloc165, gpt_neox_layers_23_mlp_dense_4h_to_h_bias1, alloc166) R.vm.kill_object(alloc165) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_weight1) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_q_scale1) R.vm.kill_object(gpt_neox_layers_23_mlp_dense_4h_to_h_bias1) gpt_neox_layers_23_attention_dense_q_weight1: R.Tensor((1024, 128), dtype="uint32") = packed_params[377] gpt_neox_layers_23_attention_dense_q_scale1: R.Tensor((1024, 32), dtype="float16") = packed_params[378] gpt_neox_layers_23_attention_dense_bias1: R.Tensor((1024,), dtype="float16") = packed_params[379] gv267: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc167: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv267, R.dtype("float16")) R.vm.kill_object(storage3) cls.gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9(gpt_neox_layers_23_attention_dense_q_weight1, gpt_neox_layers_23_attention_dense_q_scale1, reshape95, gpt_neox_layers_23_attention_dense_bias1, alloc166, alloc160, alloc167) R.vm.kill_object(alloc160) R.vm.kill_object(reshape95) R.vm.kill_object(alloc166) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_weight1) R.vm.kill_object(gpt_neox_layers_23_attention_dense_q_scale1) R.vm.kill_object(gpt_neox_layers_23_attention_dense_bias1) gpt_neox_final_layer_norm_weight1: R.Tensor((1024,), dtype="float16") = packed_params[386] gpt_neox_final_layer_norm_bias1: R.Tensor((1024,), dtype="float16") = packed_params[387] gv268: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1024), sinfo_args=(R.Shape(ndim=3),)) alloc168: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage, R.prim_value(0), gv268, R.dtype("float16")) R.vm.kill_object(storage) cls.gpt_neox_q4f16_1_layer_norm1(alloc167, gpt_neox_final_layer_norm_weight1, gpt_neox_final_layer_norm_bias1, alloc168) R.vm.kill_object(alloc167) R.vm.kill_object(gpt_neox_final_layer_norm_weight1) R.vm.kill_object(gpt_neox_final_layer_norm_bias1) storage4: R.Object = R.vm.alloc_storage(R.shape([2048]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc169: R.Tensor((1, 1, 1024), dtype="float16") = R.vm.alloc_tensor(storage4, R.prim_value(0), R.shape([1, 1, 1024]), R.dtype("float16")) R.vm.kill_object(storage4) cls.gpt_neox_q4f16_1_index(alloc168, alloc169) R.vm.kill_object(alloc168) embed_out_q_weight1: R.Tensor((vocab_size, 128), dtype="uint32") = packed_params[388] embed_out_q_scale1: R.Tensor((vocab_size, 32), dtype="float16") = packed_params[389] gv269: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage5: R.Object = R.vm.alloc_storage(gv269, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv270: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc170: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv270, R.dtype("float32")) R.vm.kill_object(storage5) cls.gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8(embed_out_q_weight1, embed_out_q_scale1, alloc169, alloc170) R.vm.kill_object(alloc169) R.vm.kill_object(embed_out_q_weight1) R.vm.kill_object(embed_out_q_scale1) gv1_1: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc170, paged_kv_cache R.vm.kill_object(alloc170) gv271: R.Tensor(dtype="float32", ndim=3) = gv1_1[0] R.call_packed("vm.builtin.match_shape", gv271, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=prefill, loc=return, annotation=R.Tuple(R.Tensor((1, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv1_1 @R.function def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64(is_size_var=True) vocab_size = T.int64(is_size_var=True) R.func_attr({"relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", logits, R.prim_value(3), R.dtype("float32"), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[0], param=logits, annotation=R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tensor_info", temperature, R.prim_value(1), R.dtype("float32"), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[1], param=temperature, annotation=R.Tensor((batch_size,), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", logits, shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[0], param=logits, annotation=R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", temperature, shape_heap, R.prim_value(1), R.prim_value(3), R.prim_value(0), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[1], param=temperature, annotation=R.Tensor((batch_size,), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) cls.gpt_neox_q4f16_1_shape_func3(shape_heap) gv557: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=2),)) lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_packed("vm.builtin.reshape", logits, gv557, sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) gv558: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage17: R.Object = R.vm.alloc_storage(gv558, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv559: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=2),)) alloc512: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv559, R.dtype("float32")) R.vm.kill_object(storage17) gv560: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage18: R.Object = R.vm.alloc_storage(gv560, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv561: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=2),)) alloc513: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv561, R.dtype("float32")) R.vm.kill_object(storage18) cls.gpt_neox_q4f16_1_chunk_lse(lv, temperature, alloc512, alloc513) lv1: R.Tuple(R.Tensor(dtype="float32", ndim=2), R.Tensor(dtype="float32", ndim=2)) = alloc512, alloc513 gv562: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=1),)) storage19: R.Object = R.vm.alloc_storage(gv562, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv563: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=2),)) alloc514: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv563, R.dtype("float32")) R.vm.kill_object(storage19) cls.gpt_neox_q4f16_1_softmax_with_chunked_sum(lv, temperature, alloc512, alloc513, alloc514) R.vm.kill_object(lv) R.vm.kill_object(alloc512) R.vm.kill_object(alloc513) gv564: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_packed("vm.builtin.reshape", alloc514, gv564, sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) R.vm.kill_object(alloc514) return gv