diff --git "a/ir_dump/debug-phase0.py" "b/ir_dump/debug-phase0.py" new file mode 100644--- /dev/null +++ "b/ir_dump/debug-phase0.py" @@ -0,0 +1,6631 @@ +# from tvm.script import ir as I +# from tvm.script import tir as T +# from tvm.script import relax as R + +@I.ir_module +class Module: + I.module_attrs({"system_lib_prefix": "gpt_neox_q0f16_"}) + @T.prim_func + def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) + batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) + logits = T.match_buffer(var_logits, (batch_size, vocab_size)) + num_seq = T.int32(is_size_var=True) + seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") + bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") + # with T.block("root"): + for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): + for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): + with T.block("block"): + vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) + vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) + T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) + T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) + T.writes(logits[seq_ids[vs], vv]) + logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) + + @T.prim_func + def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) + batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) + logits = T.match_buffer(var_logits, (batch_size, vocab_size)) + num_token = T.int32(is_size_var=True) + pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") + token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") + logit_bias = T.match_buffer(var_logit_bias, (num_token,)) + # with T.block("root"): + for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): + for p1 in T.thread_binding(256, thread="threadIdx.x"): + with T.block("block"): + vp = T.axis.spatial(num_token, p0 * 256 + p1) + T.where(p0 * 256 + p1 < num_token) + T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) + T.writes(logits[pos2seq_id[vp], token_ids[vp]]) + logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] + + @T.prim_func + def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) + batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) + logits = T.match_buffer(var_logits, (batch_size, vocab_size)) + num_seq = T.int32(is_size_var=True) + seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") + num_token = T.int32(is_size_var=True) + pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") + token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") + token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") + penalties = T.match_buffer(var_penalties, (num_seq, 3)) + # with T.block("root"): + for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): + for p1 in T.thread_binding(256, thread="threadIdx.x"): + with T.block("block"): + vp = T.axis.spatial(num_token, p0 * 256 + p1) + T.where(p0 * 256 + p1 < num_token) + T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) + T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) + logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) + logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) + + @T.prim_func + def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + B = T.int32(is_size_var=True) + Q = T.match_buffer(Q_handle, (B, 16, 128), "float16") + max_num_pages = T.int32(is_size_var=True) + pages = T.match_buffer(pages_handle, (max_num_pages, 2, 16, 16, 128), "float16", offset_factor=1) + page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) + nnz_pages = T.int32(is_size_var=True) + page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) + length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) + k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) + output = T.match_buffer(output_handle, (B, 16, 128), "float16") + lse = T.match_buffer(lse_handle, (B, 16)) + # with T.block("root"): + sm_scale: T.float32 = T.float32(0.12751743082459868) + for bx in T.thread_binding(B, thread="blockIdx.x"): + for fused_by_bz in T.thread_binding(16, thread="blockIdx.y"): + for ty in T.thread_binding(1, thread="threadIdx.y"): + for tx in T.thread_binding(32, thread="threadIdx.x"): + for tz in T.thread_binding(8, thread="threadIdx.z"): + with T.block("attn"): + T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 16 + ty + fused_by_bz % 16, tx * 4 - 64:tx * 4 - 64 + 132]) + T.writes(output[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty]) + Q_local = T.alloc_buffer((4,), "float16", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + K_smem = T.alloc_buffer((8, 128), "float16", scope="shared") + V_smem = T.alloc_buffer((8, 128), "float16", scope="shared") + O_allreduce = T.alloc_buffer((8, 1, 128), scope="shared") + md_allreduce = T.alloc_buffer((8, 1, 2), scope="shared") + S_reduce_local = T.alloc_buffer((1,), scope="local") + t0 = T.alloc_buffer((1,), scope="local") + S_local = T.alloc_buffer((1,), scope="local") + QK_local = T.alloc_buffer((4,), scope="local") + V_local = T.alloc_buffer((4,), "float16", scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_prev = T.alloc_buffer((1,), scope="local") + other_m = T.alloc_buffer((1,), scope="local") + other_d = T.alloc_buffer((1,), scope="local") + exp_mprev = T.alloc_buffer((1,), scope="local") + exp_otherm = T.alloc_buffer((1,), scope="local") + other_o = T.alloc_buffer((4,), scope="local") + st_m = T.alloc_buffer((1,), scope="local") + st_d = T.alloc_buffer((1,), scope="local") + O_local = T.alloc_buffer((4,), scope="local") + by: T.int32 = fused_by_bz % 16 + bz: T.int32 = fused_by_bz // 16 + batch_idx: T.int32 = bx + cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] + cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] + kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) + st_m[0] = T.float32(-50000.0) + st_d[0] = T.float32(1.0) + for vec in T.vectorized(4): + O_local[vec] = T.float32(0.0) + for vec in T.vectorized(4): + freq = T.float32() + Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 64, Q[bx, by + bz + ty, tx * 4 + vec + 64] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 64]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 128) / T.float32(128.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) + for iterator in range((kv_chunk_len[0] + 7) // 8): + tile_start_s: T.int32 = tz + ty + tile_start_g: T.int32 = iterator * 8 + tz + ty + for j in range(1): + with T.block("KV_load"): + T.reads() + T.writes() + row_g: T.int32 = tile_start_g + j + if row_g < kv_chunk_len[0]: + seq_offset: T.int32 = row_g + page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] + page_offset: T.int32 = seq_offset % 16 + for vec in T.vectorized(4): + freq = T.float32() + K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 64, pages[page_no, 0, by, page_offset, tx * 4 + vec + 64] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 64]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 128) / T.float32(128.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) + V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] + else: + for vec in T.vectorized(4): + K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) + V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) + T.tvm_storage_sync("shared") + m_prev[0] = st_m[0] + for j in range(1): + for vec in T.vectorized(4): + QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale + S_reduce_local[0] = T.float32(0.0) + for vec in T.unroll(4): + S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] + with T.block("block_cross_thread"): + T.reads(S_reduce_local[0]) + T.writes(t0[0]) + T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) + T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) + S_local[j] = T.float32(-50000.0) + if iterator * 8 + tz + j < kv_chunk_len[0]: + S_local[j] = t0[0] + st_m[0] = T.max(st_m[0], S_local[j]) + o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) + st_d[0] = st_d[0] * o_scale + for j in range(1): + S_local[j] = T.exp2(S_local[j] - st_m[0]) + st_d[0] = st_d[0] + S_local[j] + for j in T.vectorized(4): + O_local[j] = O_local[j] * o_scale + for j in range(1): + for vec in T.vectorized(4): + V_local[vec] = V_smem[tz + j, tx * 4 + vec] + for vec in T.vectorized(4): + O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] + for vec in T.vectorized(4): + O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] + md_allreduce[tz, ty, 0] = st_m[0] + md_allreduce[tz, ty, 1] = st_d[0] + T.tvm_storage_sync("shared") + st_m[0] = T.float32(-50000.0) + st_d[0] = T.float32(1.0) + for vec in T.vectorized(4): + O_local[vec] = T.float32(0.0) + for j in range(8): + m_prev[0] = st_m[0] + d_prev[0] = st_d[0] + other_m[0] = md_allreduce[j, ty, 0] + other_d[0] = md_allreduce[j, ty, 1] + for vec in T.vectorized(4): + other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] + st_m[0] = T.max(st_m[0], other_m[0]) + st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) + exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) + exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) + for vec in T.vectorized(4): + O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] + for vec in T.vectorized(4): + O_local[vec] = O_local[vec] / st_d[0] + for vec in T.vectorized(4): + output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) + lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) + + @T.prim_func + def batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + B = T.int32(is_size_var=True) + Q = T.match_buffer(Q_handle, (B, 16, 128), "float16") + max_num_pages = T.int32(is_size_var=True) + pages = T.match_buffer(pages_handle, (max_num_pages, 2, 16, 16, 128), "float16", offset_factor=1) + page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) + nnz_pages = T.int32(is_size_var=True) + page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) + length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) + k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) + output = T.match_buffer(output_handle, (B, 16, 128), "float16") + lse = T.match_buffer(lse_handle, (B, 16)) + # with T.block("root"): + sm_scale: T.float32 = T.float32(0.12751743082459868) + for bx in T.thread_binding(B, thread="blockIdx.x"): + for fused_by_bz in T.thread_binding(16, thread="blockIdx.y"): + for ty in T.thread_binding(1, thread="threadIdx.y"): + for tx in T.thread_binding(32, thread="threadIdx.x"): + for tz in T.thread_binding(8, thread="threadIdx.z"): + with T.block("attn"): + T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz // 16 + ty + fused_by_bz % 16, tx * 4 - 64:tx * 4 - 64 + 132]) + T.writes(output[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 16 + fused_by_bz // 16 + ty]) + Q_local = T.alloc_buffer((4,), "float16", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + K_smem = T.alloc_buffer((8, 128), "float16", scope="shared") + V_smem = T.alloc_buffer((8, 128), "float16", scope="shared") + O_allreduce = T.alloc_buffer((8, 1, 128), scope="shared") + md_allreduce = T.alloc_buffer((8, 1, 2), scope="shared") + S_reduce_local = T.alloc_buffer((1,), scope="local") + t0 = T.alloc_buffer((1,), scope="local") + S_local = T.alloc_buffer((1,), scope="local") + QK_local = T.alloc_buffer((4,), scope="local") + V_local = T.alloc_buffer((4,), "float16", scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_prev = T.alloc_buffer((1,), scope="local") + other_m = T.alloc_buffer((1,), scope="local") + other_d = T.alloc_buffer((1,), scope="local") + exp_mprev = T.alloc_buffer((1,), scope="local") + exp_otherm = T.alloc_buffer((1,), scope="local") + other_o = T.alloc_buffer((4,), scope="local") + st_m = T.alloc_buffer((1,), scope="local") + st_d = T.alloc_buffer((1,), scope="local") + O_local = T.alloc_buffer((4,), scope="local") + by: T.int32 = fused_by_bz % 16 + bz: T.int32 = fused_by_bz // 16 + batch_idx: T.int32 = bx + cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] + cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] + kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) + st_m[0] = T.float32(-50000.0) + st_d[0] = T.float32(1.0) + for vec in T.vectorized(4): + O_local[vec] = T.float32(0.0) + for vec in T.vectorized(4): + freq = T.float32() + Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 64, Q[bx, by + bz + ty, tx * 4 + vec + 64] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 64]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 128) / T.float32(128.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) + for iterator in range((kv_chunk_len[0] + 7) // 8): + tile_start_s: T.int32 = tz + ty + tile_start_g: T.int32 = iterator * 8 + tz + ty + for j in range(1): + with T.block("KV_load"): + T.reads() + T.writes() + row_g: T.int32 = tile_start_g + j + if row_g < kv_chunk_len[0]: + seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) + page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] + page_offset: T.int32 = seq_offset % 16 + for vec in T.vectorized(4): + freq = T.float32() + K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 64, pages[page_no, 0, by, page_offset, tx * 4 + vec + 64] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 64]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 128) / T.float32(128.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) + V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] + else: + for vec in T.vectorized(4): + K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) + V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) + T.tvm_storage_sync("shared") + m_prev[0] = st_m[0] + for j in range(1): + for vec in T.vectorized(4): + QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale + S_reduce_local[0] = T.float32(0.0) + for vec in T.unroll(4): + S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] + with T.block("block_cross_thread"): + T.reads(S_reduce_local[0]) + T.writes(t0[0]) + T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) + T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) + S_local[j] = T.float32(-50000.0) + if iterator * 8 + tz + j < kv_chunk_len[0]: + S_local[j] = t0[0] + st_m[0] = T.max(st_m[0], S_local[j]) + o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) + st_d[0] = st_d[0] * o_scale + for j in range(1): + S_local[j] = T.exp2(S_local[j] - st_m[0]) + st_d[0] = st_d[0] + S_local[j] + for j in T.vectorized(4): + O_local[j] = O_local[j] * o_scale + for j in range(1): + for vec in T.vectorized(4): + V_local[vec] = V_smem[tz + j, tx * 4 + vec] + for vec in T.vectorized(4): + O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] + for vec in T.vectorized(4): + O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] + md_allreduce[tz, ty, 0] = st_m[0] + md_allreduce[tz, ty, 1] = st_d[0] + T.tvm_storage_sync("shared") + st_m[0] = T.float32(-50000.0) + st_d[0] = T.float32(1.0) + for vec in T.vectorized(4): + O_local[vec] = T.float32(0.0) + for j in range(8): + m_prev[0] = st_m[0] + d_prev[0] = st_d[0] + other_m[0] = md_allreduce[j, ty, 0] + other_d[0] = md_allreduce[j, ty, 1] + for vec in T.vectorized(4): + other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] + st_m[0] = T.max(st_m[0], other_m[0]) + st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) + exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) + exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) + for vec in T.vectorized(4): + O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] + for vec in T.vectorized(4): + O_local[vec] = O_local[vec] / st_d[0] + for vec in T.vectorized(4): + output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) + lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) + + @T.prim_func + def batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + total_len = T.int32(is_size_var=True) + q = T.match_buffer(var_q, (total_len, 16, 128), "float16") + batch_size = T.int32(is_size_var=True) + q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) + max_num_pages = T.int32(is_size_var=True) + pages = T.match_buffer(var_pages, (max_num_pages, 2, 16, 16, 128), "float16", offset_factor=1) + page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) + nnz_pages = T.int32(is_size_var=True) + page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) + length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) + k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) + output = T.match_buffer(var_output, (total_len, 16, 128), "float16") + lse = T.match_buffer(var_lse, (total_len, 16)) + # with T.block("root"): + for lbx in T.thread_binding(16, thread="blockIdx.x"): + for lby in T.thread_binding(16, thread="blockIdx.y"): + for lty in T.thread_binding(4, thread="threadIdx.y"): + for ltx in T.thread_binding(32, thread="threadIdx.x"): + with T.block("attn"): + bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) + T.reads() + T.writes() + tile_id = T.alloc_buffer((1,), "int32", scope="local") + batch_idx = T.alloc_buffer((1,), "int32", scope="local") + batch_tiles = T.alloc_buffer((1,), "int32", scope="local") + batch_rows = T.alloc_buffer((1,), "int32", scope="local") + iterator = T.alloc_buffer((1,), "int32", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + Q_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + K_smem = T.alloc_buffer((16, 128), "float16", scope="shared") + V_smem = T.alloc_buffer((16, 128), "float16", scope="shared") + S_smem = T.alloc_buffer((32, 16), scope="shared") + S_local = T.alloc_buffer((32, 16), scope="local") + O_local = T.alloc_buffer((32, 128), scope="local") + m_smem = T.alloc_buffer((32,), scope="shared") + m_prev_smem = T.alloc_buffer((32,), scope="shared") + d_smem = T.alloc_buffer((32,), scope="shared") + m_new = T.alloc_buffer((1,), scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_new = T.alloc_buffer((1,), scope="local") + tile_id[0] = bx + batch_idx[0] = 0 + batch_rows[0] = q_indptr[1] - q_indptr[0] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + while T.tvm_thread_invariant(batch_idx[0] < batch_size): + while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: + tile_id[0] = tile_id[0] - batch_tiles[0] + batch_idx[0] = batch_idx[0] + 1 + if batch_idx[0] < batch_size: + b_idx: T.int32 = batch_idx[0] + batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + if T.tvm_thread_invariant(batch_idx[0] < batch_size): + b_idx: T.int32 = batch_idx[0] + LH_start: T.int32 = tile_id[0] * 32 + q_indptr_val: T.int32 = q_indptr[b_idx] + cur_page_indptr_begin: T.int32 = page_indptr[b_idx] + cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] + kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + m_smem[row] = T.float32(-50000.0) + d_smem[row] = T.float32(1.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = T.float32(0.0) + T.tvm_storage_sync("shared") + for li_lj_fused_0 in range(8): + for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for li_lj_fused_3 in T.vectorized(4): + with T.block("Q_load"): + i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 128) + j = T.axis.spatial(128, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = q_indptr_val + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + freq = T.float32() + Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, q[cur_L, cur_H_qo, j + 64] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 64]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), q[cur_L, cur_H_qo, j]) + else: + Q_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for iterator_1 in range((kv_chunk_len[0] + 15) // 16): + L_kv_start: T.int32 = iterator_1 * 16 + for lz_ly_fused_0 in range(4): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("K_load"): + i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + seq_offset: T.int32 = cur_L + page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] + page_offset: T.int32 = seq_offset % 16 + freq = T.float32() + K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, pages[page_no, 0, by, page_offset, j + 64] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 64]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), pages[page_no, 0, by, page_offset, j]) + else: + K_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for lz_ly_fused_0 in range(4): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("V_load"): + i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + seq_offset: T.int32 = cur_L + page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] + page_offset: T.int32 = seq_offset % 16 + V_smem[i, j] = pages[page_no, 1, by, page_offset, j] + else: + V_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(Q_smem[0:32, 0:128], K_smem[0:16, 0:128]) + T.writes(S_local[0:32, 0:16]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(2, 2): + with T.block("S_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) + j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) + T.reads() + T.writes(S_local[i, j]) + S_local[i, j] = T.float32(0.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, li_1, lj_1, lk_1 in T.grid(16, 2, 2, 8): + with T.block("S_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) + k = T.axis.reduce(128, lk_0 * 8 + lk_1) + T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) + T.writes(S_local[i, j]) + S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.12751743082459868) + T.tvm_storage_sync("shared") + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(2, 2): + with T.block("S_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) + T.reads(S_local[i, j]) + T.writes(S_smem[i, j]) + S_smem[i, j] = S_local[i, j] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update1"): + T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) + T.writes(m_prev[i], m_new[i], d_new[i]) + m_prev[i] = m_smem[row] + m_new[i] = m_smem[row] + row_: T.int32 = LH_start + row + for j in range(16): + if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): + m_new[i] = T.max(m_new[i], S_smem[row, j]) + d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + with T.block("update"): + T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) + T.writes(S_smem[row, 0:16]) + for j in range(16): + if row < 32: + row_: T.int32 = LH_start + row + if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): + S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) + else: + S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update"): + T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) + T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) + for j in range(16): + d_new[i] = d_new[i] + S_smem[row, j] + m_smem[row] = m_new[i] + d_smem[row] = d_new[i] + m_prev_smem[row] = m_prev[i] + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:128]) + T.writes(O_local[0:32, 0:128]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(4, 8): + with T.block("O_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) + j = T.axis.spatial(128, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 8 + lj_1_init) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): + with T.block("O_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + k = T.axis.reduce(16, lk_0 * 8 + lk_1) + T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) + T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) + for li_0 in range(1): + for li_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_2 in T.thread_binding(32, thread="threadIdx.x"): + with T.block("lse_store"): + i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) + T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) + T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) + T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) + tile_id[0] = tile_id[0] + 16 + + @T.prim_func + def batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + total_len = T.int32(is_size_var=True) + q = T.match_buffer(var_q, (total_len, 16, 128), "float16") + batch_size = T.int32(is_size_var=True) + q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) + max_num_pages = T.int32(is_size_var=True) + pages = T.match_buffer(var_pages, (max_num_pages, 2, 16, 16, 128), "float16", offset_factor=1) + page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) + nnz_pages = T.int32(is_size_var=True) + page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) + length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) + k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) + output = T.match_buffer(var_output, (total_len, 16, 128), "float16") + lse = T.match_buffer(var_lse, (total_len, 16)) + # with T.block("root"): + for lbx in T.thread_binding(16, thread="blockIdx.x"): + for lby in T.thread_binding(16, thread="blockIdx.y"): + for lty in T.thread_binding(4, thread="threadIdx.y"): + for ltx in T.thread_binding(32, thread="threadIdx.x"): + with T.block("attn"): + bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) + T.reads() + T.writes() + tile_id = T.alloc_buffer((1,), "int32", scope="local") + batch_idx = T.alloc_buffer((1,), "int32", scope="local") + batch_tiles = T.alloc_buffer((1,), "int32", scope="local") + batch_rows = T.alloc_buffer((1,), "int32", scope="local") + iterator = T.alloc_buffer((1,), "int32", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + Q_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + K_smem = T.alloc_buffer((16, 128), "float16", scope="shared") + V_smem = T.alloc_buffer((16, 128), "float16", scope="shared") + S_smem = T.alloc_buffer((32, 16), scope="shared") + S_local = T.alloc_buffer((32, 16), scope="local") + O_local = T.alloc_buffer((32, 128), scope="local") + m_smem = T.alloc_buffer((32,), scope="shared") + m_prev_smem = T.alloc_buffer((32,), scope="shared") + d_smem = T.alloc_buffer((32,), scope="shared") + m_new = T.alloc_buffer((1,), scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_new = T.alloc_buffer((1,), scope="local") + tile_id[0] = bx + batch_idx[0] = 0 + batch_rows[0] = q_indptr[1] - q_indptr[0] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + while T.tvm_thread_invariant(batch_idx[0] < batch_size): + while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: + tile_id[0] = tile_id[0] - batch_tiles[0] + batch_idx[0] = batch_idx[0] + 1 + if batch_idx[0] < batch_size: + b_idx: T.int32 = batch_idx[0] + batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + if T.tvm_thread_invariant(batch_idx[0] < batch_size): + b_idx: T.int32 = batch_idx[0] + LH_start: T.int32 = tile_id[0] * 32 + q_indptr_val: T.int32 = q_indptr[b_idx] + cur_page_indptr_begin: T.int32 = page_indptr[b_idx] + cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] + kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + m_smem[row] = T.float32(-50000.0) + d_smem[row] = T.float32(1.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = T.float32(0.0) + T.tvm_storage_sync("shared") + for li_lj_fused_0 in range(8): + for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for li_lj_fused_3 in T.vectorized(4): + with T.block("Q_load"): + i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 128) + j = T.axis.spatial(128, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = q_indptr_val + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + freq = T.float32() + Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, q[cur_L, cur_H_qo, j + 64] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 64]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), q[cur_L, cur_H_qo, j]) + else: + Q_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for iterator_1 in range((kv_chunk_len[0] + 15) // 16): + L_kv_start: T.int32 = iterator_1 * 16 + for lz_ly_fused_0 in range(4): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("K_load"): + i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) + page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] + page_offset: T.int32 = seq_offset % 16 + freq = T.float32() + K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, pages[page_no, 0, by, page_offset, j + 64] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 64]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), pages[page_no, 0, by, page_offset, j]) + else: + K_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for lz_ly_fused_0 in range(4): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("V_load"): + i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) + page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] + page_offset: T.int32 = seq_offset % 16 + V_smem[i, j] = pages[page_no, 1, by, page_offset, j] + else: + V_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(Q_smem[0:32, 0:128], K_smem[0:16, 0:128]) + T.writes(S_local[0:32, 0:16]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(2, 2): + with T.block("S_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) + j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) + T.reads() + T.writes(S_local[i, j]) + S_local[i, j] = T.float32(0.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, li_1, lj_1, lk_1 in T.grid(16, 2, 2, 8): + with T.block("S_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) + k = T.axis.reduce(128, lk_0 * 8 + lk_1) + T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) + T.writes(S_local[i, j]) + S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.12751743082459868) + T.tvm_storage_sync("shared") + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(2, 2): + with T.block("S_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) + T.reads(S_local[i, j]) + T.writes(S_smem[i, j]) + S_smem[i, j] = S_local[i, j] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update1"): + T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) + T.writes(m_prev[i], m_new[i], d_new[i]) + m_prev[i] = m_smem[row] + m_new[i] = m_smem[row] + row_: T.int32 = LH_start + row + for j in range(16): + if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): + m_new[i] = T.max(m_new[i], S_smem[row, j]) + d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + with T.block("update"): + T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) + T.writes(S_smem[row, 0:16]) + for j in range(16): + if row < 32: + row_: T.int32 = LH_start + row + if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): + S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) + else: + S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update"): + T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) + T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) + for j in range(16): + d_new[i] = d_new[i] + S_smem[row, j] + m_smem[row] = m_new[i] + d_smem[row] = d_new[i] + m_prev_smem[row] = m_prev[i] + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:128]) + T.writes(O_local[0:32, 0:128]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(4, 8): + with T.block("O_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) + j = T.axis.spatial(128, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 8 + lj_1_init) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): + with T.block("O_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + k = T.axis.reduce(16, lk_0 * 8 + lk_1) + T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) + T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) + for li_0 in range(1): + for li_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_2 in T.thread_binding(32, thread="threadIdx.x"): + with T.block("lse_store"): + i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) + T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) + T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) + T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) + tile_id[0] = tile_id[0] + 16 + + @T.prim_func + def batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + qo_len = T.int32(is_size_var=True) + q = T.match_buffer(var_q, (qo_len, 16, 128), "float16") + batch_size = T.int32(is_size_var=True) + q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) + kv_len = T.int32(is_size_var=True) + k = T.match_buffer(var_k, (kv_len, 16, 128), "float16") + v = T.match_buffer(var_v, (kv_len, 16, 128), "float16") + kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) + k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) + output = T.match_buffer(var_output, (qo_len, 16, 128), "float16") + lse = T.match_buffer(var_lse, (qo_len, 16)) + # with T.block("root"): + for lbx in T.thread_binding(8, thread="blockIdx.x"): + for lby in T.thread_binding(16, thread="blockIdx.y"): + for lty in T.thread_binding(4, thread="threadIdx.y"): + for ltx in T.thread_binding(32, thread="threadIdx.x"): + with T.block("attn"): + bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) + T.reads() + T.writes() + tile_id = T.alloc_buffer((1,), "int32", scope="local") + batch_idx = T.alloc_buffer((1,), "int32", scope="local") + batch_tiles = T.alloc_buffer((1,), "int32", scope="local") + batch_rows = T.alloc_buffer((1,), "int32", scope="local") + iterator = T.alloc_buffer((1,), "int32", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + Q_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + K_smem = T.alloc_buffer((128, 32), "float16", scope="shared") + V_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + S_smem = T.alloc_buffer((32, 32), scope="shared") + S_local = T.alloc_buffer((32, 32), scope="local") + O_local = T.alloc_buffer((32, 128), scope="local") + m_smem = T.alloc_buffer((32,), scope="shared") + m_prev_smem = T.alloc_buffer((32,), scope="shared") + d_smem = T.alloc_buffer((32,), scope="shared") + m_new = T.alloc_buffer((1,), scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_new = T.alloc_buffer((1,), scope="local") + tile_id[0] = bx + batch_idx[0] = 0 + batch_rows[0] = q_indptr[1] - q_indptr[0] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + while T.tvm_thread_invariant(batch_idx[0] < batch_size): + while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: + tile_id[0] = tile_id[0] - batch_tiles[0] + batch_idx[0] = batch_idx[0] + 1 + if batch_idx[0] < batch_size: + b_idx: T.int32 = batch_idx[0] + batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + if T.tvm_thread_invariant(batch_idx[0] < batch_size): + b_idx: T.int32 = batch_idx[0] + q_indptr_val: T.int32 = q_indptr[b_idx] + LH_start: T.int32 = tile_id[0] * 32 + kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + m_smem[row] = T.float32(-50000.0) + d_smem[row] = T.float32(1.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1 in range(4): + for lj_1_0 in T.unroll(1): + for lj_1_1 in T.vectorized(8): + with T.block("O_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1_0 * 8 + lj_1_1) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = T.float32(0.0) + T.tvm_storage_sync("shared") + for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_0, lj_0_0 in T.grid(2, 2): + for lj_1 in T.vectorized(8): + with T.block("Q_load"): + i = T.axis.spatial(32, li_0 * 16 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 8) + j = T.axis.spatial(128, lj_0_0 * 64 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 8 * 8 + lj_1) + T.reads() + T.writes() + cur_L: T.int32 = q_indptr_val + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + freq = T.float32() + Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, q[cur_L, cur_H_qo, j + 64] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 64]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), q[cur_L, cur_H_qo, j]) + else: + Q_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for iterator_1 in range((kv_chunk_len[0] + 31) // 32): + L_kv_start: T.int32 = iterator_1 * 32 + L_kv_base: T.int32 = kv_indptr[b_idx] + for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lz_0, ly_0_0 in T.grid(2, 2): + for ly_1 in T.vectorized(8): + with T.block("K_load"): + i = T.axis.spatial(32, lz_0 * 16 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 8) + j = T.axis.spatial(128, ly_0_0 * 64 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 8 * 8 + ly_1) + T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 64:j - 64 + 129]) + T.writes(K_smem[j, i]) + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + freq = T.float32() + K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, k[L_kv_base + cur_L, by, j + 64] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 64]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), k[L_kv_base + cur_L, by, j]) + else: + K_smem[j, i] = T.float16(0.0) + T.tvm_storage_sync("shared") + for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lz_0, ly_0_0 in T.grid(2, 2): + for ly_1 in T.vectorized(8): + with T.block("V_load"): + i = T.axis.spatial(32, lz_0 * 16 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 8) + j = T.axis.spatial(128, ly_0_0 * 64 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 8 * 8 + ly_1) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + V_smem[i, j] = v[L_kv_base + cur_L, by, j] + else: + V_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(Q_smem[0:32, 0:128], K_smem[0:128, 0:32]) + T.writes(S_local[0:32, 0:32]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init in T.unroll(2): + for lj_1_0_init in T.unroll(1): + for lj_1_1_init in T.vectorized(4): + with T.block("S_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) + j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_0_init * 4 + lj_1_1_init) + T.reads() + T.writes(S_local[i, j]) + S_local[i, j] = T.float32(0.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0 in range(8): + for li_1 in T.unroll(2): + for lj_1_0 in T.unroll(1): + for lj_1_1 in T.vectorized(4): + for lk_1 in range(16): + with T.block("S_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) + k_1 = T.axis.reduce(128, lk_0 * 16 + lk_1) + T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) + T.writes(S_local[i, j]) + S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.12751743082459868) + T.tvm_storage_sync("shared") + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1 in range(2): + for lj_1_0 in T.unroll(1): + for lj_1_1 in T.vectorized(4): + with T.block("S_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) + T.reads(S_local[i, j]) + T.writes(S_smem[i, j]) + S_smem[i, j] = S_local[i, j] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update1"): + T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) + T.writes(m_prev[i], m_new[i], d_new[i]) + m_prev[i] = m_smem[row] + m_new[i] = m_smem[row] + row_: T.int32 = LH_start + row + for j in range(32): + if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): + m_new[i] = T.max(m_new[i], S_smem[row, j]) + d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + with T.block("update"): + T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) + T.writes(S_smem[row, 0:32]) + for j in range(32): + if row < 32: + row_: T.int32 = LH_start + row + if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): + S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) + else: + S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update"): + T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) + T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) + for j in range(32): + d_new[i] = d_new[i] + S_smem[row, j] + m_smem[row] = m_new[i] + d_smem[row] = d_new[i] + m_prev_smem[row] = m_prev[i] + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:128]) + T.writes(O_local[0:32, 0:128]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init in T.unroll(4): + for lj_1_0_init in T.unroll(1): + for lj_1_1_init in T.vectorized(8): + with T.block("O_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) + j = T.axis.spatial(128, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 8 + lj_1_0_init * 8 + lj_1_1_init) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, lk_1 in T.grid(2, 16): + for li_1 in T.unroll(4): + for lj_1_0 in T.unroll(1): + for lj_1_1 in T.vectorized(8): + with T.block("O_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1_0 * 8 + lj_1_1) + k_1 = T.axis.reduce(32, lk_0 * 16 + lk_1) + T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1 in range(4): + for lj_1_0 in T.unroll(1): + for lj_1_1 in T.vectorized(8): + with T.block("O_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1_0 * 8 + lj_1_1) + T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) + T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) + for li_0 in range(1): + for li_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_2 in T.thread_binding(32, thread="threadIdx.x"): + with T.block("lse_store"): + i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) + T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) + T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) + T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) + tile_id[0] = tile_id[0] + 8 + + @T.prim_func + def batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + qo_len = T.int32(is_size_var=True) + q = T.match_buffer(var_q, (qo_len, 16, 128), "float16") + q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) + kv_len = T.int32(is_size_var=True) + k = T.match_buffer(var_k, (kv_len, 16, 128), "float16") + v = T.match_buffer(var_v, (kv_len, 16, 128), "float16") + kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) + mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) + tree_size = T.int32(is_size_var=True) + mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) + output = T.match_buffer(var_output, (qo_len, 16, 128), "float16") + lse = T.match_buffer(var_lse, (qo_len, 16)) + # with T.block("root"): + for lbx in T.thread_binding(16, thread="blockIdx.x"): + for lby in T.thread_binding(16, thread="blockIdx.y"): + for lty in T.thread_binding(4, thread="threadIdx.y"): + for ltx in T.thread_binding(32, thread="threadIdx.x"): + with T.block("attn"): + bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) + T.reads() + T.writes() + tile_id = T.alloc_buffer((1,), "int32", scope="local") + batch_idx = T.alloc_buffer((1,), "int32", scope="local") + batch_tiles = T.alloc_buffer((1,), "int32", scope="local") + batch_rows = T.alloc_buffer((1,), "int32", scope="local") + iterator = T.alloc_buffer((1,), "int32", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + Q_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + K_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + V_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + S_smem = T.alloc_buffer((32, 32), scope="shared") + S_local = T.alloc_buffer((32, 32), scope="local") + O_local = T.alloc_buffer((32, 128), scope="local") + m_smem = T.alloc_buffer((32,), scope="shared") + m_prev_smem = T.alloc_buffer((32,), scope="shared") + d_smem = T.alloc_buffer((32,), scope="shared") + m_new = T.alloc_buffer((1,), scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_new = T.alloc_buffer((1,), scope="local") + tile_id[0] = bx + batch_idx[0] = 0 + batch_rows[0] = q_indptr[1] - q_indptr[0] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + while T.tvm_thread_invariant(batch_idx[0] < batch_size): + while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: + tile_id[0] = tile_id[0] - batch_tiles[0] + batch_idx[0] = batch_idx[0] + 1 + if batch_idx[0] < batch_size: + b_idx: T.int32 = batch_idx[0] + batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + if T.tvm_thread_invariant(batch_idx[0] < batch_size): + b_idx: T.int32 = batch_idx[0] + LH_start: T.int32 = tile_id[0] * 32 + q_indptr_val: T.int32 = q_indptr[b_idx] + kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + m_smem[row] = T.float32(-50000.0) + d_smem[row] = T.float32(1.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = T.float32(0.0) + T.tvm_storage_sync("shared") + for li_lj_fused_0 in range(8): + for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for li_lj_fused_3 in T.vectorized(4): + with T.block("Q_load"): + i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 128) + j = T.axis.spatial(128, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = q_indptr_val + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + freq = T.float32() + Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, q[cur_L, cur_H_qo, j + 64] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 64]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), q[cur_L, cur_H_qo, j]) + else: + Q_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for iterator_1 in range((kv_chunk_len[0] + 31) // 32): + L_kv_start: T.int32 = iterator_1 * 32 + L_kv_base: T.int32 = kv_indptr[b_idx] + for lz_ly_fused_0 in range(8): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("KV_load"): + i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_base + L_kv_start + i + if L_kv_start + i < kv_chunk_len[0]: + freq = T.float32() + K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, k[cur_L, by, j + 64] * T.float16(-1.0), k[cur_L, by, j - 64]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), k[cur_L, by, j]) + V_smem[i, j] = v[cur_L, by, j] + else: + K_smem[i, j] = T.float16(0.0) + V_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(Q_smem[0:32, 0:128], K_smem[0:32, 0:128]) + T.writes(S_local[0:32, 0:32]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(2, 4): + with T.block("S_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) + j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) + T.reads() + T.writes(S_local[i, j]) + S_local[i, j] = T.float32(0.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, li_1, lj_1, lk_1 in T.grid(16, 2, 4, 8): + with T.block("S_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) + k_1 = T.axis.reduce(128, lk_0 * 8 + lk_1) + T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) + T.writes(S_local[i, j]) + S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.12751743082459868) + T.tvm_storage_sync("shared") + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(2, 4): + with T.block("S_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) + T.reads(S_local[i, j]) + T.writes(S_smem[i, j]) + S_smem[i, j] = S_local[i, j] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update1"): + T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) + T.writes(m_prev[i], m_new[i], d_new[i]) + m_prev[i] = m_smem[row] + m_new[i] = m_smem[row] + row_: T.int32 = LH_start + row + for j in range(32): + if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): + m_new[i] = T.max(m_new[i], S_smem[row, j]) + d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + with T.block("update"): + T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) + T.writes(S_smem[row, 0:32]) + for j in range(32): + if row < 32: + row_: T.int32 = LH_start + row + if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): + S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) + else: + S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update"): + T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) + T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) + for j in range(32): + d_new[i] = d_new[i] + S_smem[row, j] + m_smem[row] = m_new[i] + d_smem[row] = d_new[i] + m_prev_smem[row] = m_prev[i] + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:128]) + T.writes(O_local[0:32, 0:128]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(4, 8): + with T.block("O_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) + j = T.axis.spatial(128, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 8 + lj_1_init) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): + with T.block("O_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) + T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) + T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) + for li_0 in range(1): + for li_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_2 in T.thread_binding(32, thread="threadIdx.x"): + with T.block("lse_store"): + i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) + T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) + T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) + T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) + tile_id[0] = tile_id[0] + 16 + + @T.prim_func + def chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) + A = T.match_buffer(var_A, (batch_size, vocab_size)) + temperature = T.match_buffer(var_temperature, (batch_size,)) + num_chunks = T.int64(is_size_var=True) + chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) + chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) + # with T.block("root"): + A_pad = T.alloc_buffer((batch_size, num_chunks, T.int64(4096))) + temp_max = T.alloc_buffer((batch_size, num_chunks)) + temp_sum = T.alloc_buffer((batch_size, num_chunks)) + for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): + with T.block("pad"): + v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) + T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2]) + T.writes(A_pad[v0, v1, v2]) + A_pad[v0, v1, v2] = T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * T.int64(4096) + v2] / temperature[v0], A[v0, v1 * T.int64(4096) + v2]), T.float32(-340282346638528859811704183484516925440.0)) + for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): + with T.block("max"): + v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) + T.reads(A_pad[v0, v1, v2]) + T.writes(temp_max[v0, v1]) + with T.init(): + temp_max[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) + temp_max[v0, v1] = T.max(temp_max[v0, v1], A_pad[v0, v1, v2]) + for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): + with T.block("sum_exp"): + v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) + T.reads(temperature[v0], A_pad[v0, v1, v2], temp_max[v0, v1]) + T.writes(temp_sum[v0, v1]) + with T.init(): + temp_sum[v0, v1] = T.float32(0.0) + temp_sum[v0, v1] = temp_sum[v0, v1] + T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A_pad[v0, v1, v2] - temp_max[v0, v1]), T.Cast("float32", A_pad[v0, v1, v2] == temp_max[v0, v1])), T.float32(0.0)) + for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(1)): + with T.block("log"): + v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) + T.reads(temperature[v0], temp_sum[v0, v1], temp_max[v0, v1]) + T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) + chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum[v0, v1]), temp_sum[v0, v1]) + chunked_max[v0, v1] = temp_max[v0, v1] + + @T.prim_func + def compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + num_pages = T.int32() + pages = T.match_buffer(var_pages, (num_pages, 2, 16, 16, 128), "float16", offset_factor=1) + copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) + total_copy_length = T.int32() + copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) + with T.block("root"): + T.reads() + T.writes() + for bhd_o in T.thread_binding(batch_size * 8, thread="blockIdx.x"): + for bhd_i in T.thread_binding(256, thread="threadIdx.x"): + b: T.int32 = (bhd_o * 256 + bhd_i) // 2048 + h: T.int32 = (bhd_o * 256 + bhd_i) // 128 % 16 + d: T.int32 = (bhd_o * 256 + bhd_i) % 128 + if bhd_o * 256 + bhd_i < batch_size * 16 * 128: + for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): + src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] + dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] + pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] + pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] + + @T.prim_func + def copy_single_page(var_pages: T.handle, src_page_id: T.int64, tgt_page_id: T.int64, copy_length: T.int64): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + num_pages, page_size = T.int32(), T.int64() + pages = T.match_buffer(var_pages, (num_pages, 2, 16, page_size, 128), "float16", offset_factor=1) + # with T.block("root"): + for b in T.thread_binding(copy_length * T.int64(8), thread="blockIdx.x"): + for t in T.thread_binding(256, thread="threadIdx.x"): + with T.block("copy"): + vh = T.axis.spatial(16, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) // (copy_length * T.int64(128)))) + vp = T.axis.spatial(copy_length, (b * T.int64(256) + T.Cast("int64", t)) % (copy_length * T.int64(128)) // T.int64(128)) + vd = T.axis.spatial(128, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) % T.int64(128))) + T.where(b * T.int64(256) + T.Cast("int64", t) < copy_length * T.int64(16) * T.int64(128)) + T.reads(pages[src_page_id, 0:2, vh, vp, vd]) + T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) + pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] + pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] + + @T.prim_func + def fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): + T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + seq_len = T.int32() + qkv = T.match_buffer(var_qkv, (seq_len, 48, 128), "float16") + position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) + q = T.match_buffer(var_q, (seq_len, 16, 128), "float16") + k = T.match_buffer(var_k, (seq_len, 16, 128), "float16") + v = T.match_buffer(var_v, (seq_len, 16, 128), "float16") + # with T.block("root"): + for iters_0, iters_1, iters_2 in T.grid(seq_len, 48, 128): + with T.block("llama_fused_rope"): + s, h, d = T.axis.remap("SSS", [iters_0, iters_1, iters_2]) + T.reads(position_map[s], qkv[s, h, d - 16:d - 16 + 33]) + T.writes(q[s, h, d], k[s, h - 16, d], v[s, h - 32, d]) + if h < 16: + freq = T.float32() + q[s, h, d] = T.if_then_else(apply_rope > 0 and d < 32, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 16, qkv[s, h, d + 16] * T.float16(-1.0), qkv[s, h, d - 16]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 32) / T.float32(32.0))}), qkv[s, h, d]) + else: + if h < 32: + freq = T.float32() + k[s, h - 16, d] = T.if_then_else(apply_rope > 0 and d < 32, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 16, qkv[s, h, d + 16] * T.float16(-1.0), qkv[s, h, d - 16]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 32) / T.float32(32.0))}), qkv[s, h, d]) + else: + v[s, h - 32, d] = qkv[s, h, d] + + @T.prim_func + def gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) + src = T.match_buffer(var_src, (m, n)) + batch_size = T.int32(is_size_var=True) + indices = T.match_buffer(var_indices, (batch_size,), "int32") + dst = T.match_buffer(var_dst, (batch_size, n)) + # with T.block("root"): + for b, j in T.grid(batch_size, n): + with T.block("gather_2d"): + vb, vj = T.axis.remap("SS", [b, j]) + T.reads(src[indices[vb], vj], indices[vb]) + T.writes(dst[vb, vj]) + dst[vb, vj] = src[indices[vb], vj] + + @T.prim_func(private=True) + def index(var_layer_norm48: T.handle, index: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16")): + T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + seq_len = T.int64() + layer_norm48 = T.match_buffer(var_layer_norm48, (T.int64(1), seq_len, T.int64(2048)), "float16") + # with T.block("root"): + for i, _, k in T.grid(T.int64(1), T.int64(1), T.int64(2048)): + with T.block("index"): + v_i, v__, v_k = T.axis.remap("SSS", [i, _, k]) + T.reads(layer_norm48[v_i, seq_len - T.int64(1), v_k]) + T.writes(index[v_i, v__, v_k]) + index[v_i, v__, v_k] = layer_norm48[v_i, seq_len - T.int64(1), v_k] + + @T.prim_func + def merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) + V = T.match_buffer(v, (N, H, D), "float16") + S = T.match_buffer(s, (N, H)) + V_other = T.match_buffer(v_other, (N, H, D), "float16") + S_other = T.match_buffer(s_other, (N, H)) + # with T.block("root"): + for bx in T.thread_binding(N, thread="blockIdx.x"): + for by in T.thread_binding(2, thread="blockIdx.y"): + for ty in T.thread_binding(8, thread="threadIdx.y"): + for tx in T.thread_binding(32, thread="threadIdx.x"): + with T.block("merge"): + T.reads(S[bx, ty + by * 8], S_other[bx, ty + by * 8], V[bx, ty + by * 8, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 8, tx * 4:tx * 4 + 4]) + T.writes(V[bx, ty + by * 8, tx * 4:tx * 4 + 4], S[bx, ty + by * 8]) + s_val = T.alloc_buffer((1,), scope="local") + s_other_val = T.alloc_buffer((1,), scope="local") + s_max = T.alloc_buffer((1,), scope="local") + scale = T.alloc_buffer((1,), scope="local") + other_scale = T.alloc_buffer((1,), scope="local") + v_vec = T.alloc_buffer((4,), "float16", scope="local") + v_other_vec = T.alloc_buffer((4,), "float16", scope="local") + s_val[0] = S[bx, ty + by * 8] + s_other_val[0] = S_other[bx, ty + by * 8] + s_max[0] = T.max(s_val[0], s_other_val[0]) + s_val[0] = T.exp2(s_val[0] - s_max[0]) + s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) + scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) + other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) + for vec in T.vectorized(4): + v_vec[vec] = V[bx, ty + by * 8, tx * 4 + vec] + for vec in T.vectorized(4): + v_other_vec[vec] = V_other[bx, ty + by * 8, tx * 4 + vec] + for vec in range(4): + v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) + for vec in T.vectorized(4): + V[bx, ty + by * 8, tx * 4 + vec] = v_vec[vec] + S[bx, ty + by * 8] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] + + @T.prim_func + def scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) + src = T.match_buffer(var_src, (batch_size, n)) + indices = T.match_buffer(var_indices, (batch_size,), "int32") + m = T.int32(is_size_var=True) + dst = T.match_buffer(var_dst, (m, n)) + # with T.block("root"): + for b, j in T.grid(batch_size, n): + with T.block("scatter_2d"): + vb, vj = T.axis.remap("SS", [b, j]) + T.reads(src[vb, vj], indices[vb]) + T.writes(dst[indices[vb], vj]) + dst[indices[vb], vj] = src[vb, vj] + + @T.prim_func + def softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) + batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) + A = T.match_buffer(var_A, (batch_size, vocab_size)) + temperature = T.match_buffer(var_temperature, (batch_size,)) + num_chunks = T.int64(is_size_var=True) + chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) + chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) + softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) + # with T.block("root"): + temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") + temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") + for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): + for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): + for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): + with T.block("max"): + v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) + v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) + T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) + T.reads(chunked_max[v0, v1]) + T.writes(temp_max_shared[v0]) + with T.init(): + temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) + temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) + for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): + for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): + with T.block("sum_exp"): + v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) + v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) + T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) + T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) + T.writes(temp_sum_shared[v0]) + with T.init(): + temp_sum_shared[v0] = T.float32(0.0) + temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) + for l2_0 in T.serial(T.int64(16), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): + for l2_1 in T.thread_binding(T.int64(8), thread="threadIdx.y"): + for l2_2 in T.thread_binding(T.int64(32), thread="threadIdx.x"): + with T.block("log_pad"): + v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) + v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) + v2 = T.axis.spatial(T.int64(4096), l2_0 * T.int64(256) + l2_1 * T.int64(32) + l2_2) + T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2], temp_sum_shared[v0], temp_max_shared[v0]) + T.writes(softmax[v0, v1 * T.int64(4096) + v2]) + if v1 * T.int64(4096) + v2 < vocab_size: + softmax[v0, v1 * T.int64(4096) + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * T.int64(4096) + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * T.int64(4096) + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) + + @T.prim_func + def tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int64): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + num_pages, page_size = T.int64(), T.int64(is_size_var=True) + pages = T.match_buffer(var_pages, (num_pages, 2, 16, page_size, 128), "float16", offset_factor=1) + seqlen = T.int64(is_size_var=True) + position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) + k_data = T.match_buffer(var_k_data, (24, seqlen, 16, 128), "float16") + v_data = T.match_buffer(var_v_data, (24, seqlen, 16, 128), "float16") + # with T.block("root"): + for p, h, d in T.grid(seqlen, 16, 128): + with T.block("copy0"): + vp, vh, vd = T.axis.remap("SSS", [p, h, d]) + T.reads(position_map[vp], pages[T.Cast("int64", position_map[vp]) // page_size, 0:2, vh, T.Cast("int64", position_map[vp]) % page_size, vd]) + T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) + position: T.int32 = position_map[vp] + k_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 0, vh, T.Cast("int64", position) % page_size, vd] + v_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 1, vh, T.Cast("int64", position) % page_size, vd] + + @T.prim_func + def tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) + num_pages = T.int64() + pages = T.match_buffer(var_pages, (num_pages, 2, 16, 16, 128), "float16", offset_factor=1) + ntoken = T.int64(is_size_var=True) + k_data = T.match_buffer(var_k_data, (ntoken, 16, 128), "float16") + v_data = T.match_buffer(var_v_data, (ntoken, 16, 128), "float16") + position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) + # with T.block("root"): + for global_pos, h, f in T.grid(ntoken, 16, 128): + if position_map[global_pos] != -1: + with T.block("k_transpose_append"): + vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) + T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) + T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) + position: T.int32 = position_map[vgpos] + pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] + with T.block("v_transpose_append"): + vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) + T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) + T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) + position: T.int32 = position_map[vgpos] + pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] + + @T.prim_func + def tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): + T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) + total_len = T.int32(is_size_var=True) + q = T.match_buffer(var_q, (total_len, 16, 128), "float16") + batch_size = T.int32(is_size_var=True) + q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) + max_num_pages = T.int32(is_size_var=True) + pages = T.match_buffer(var_pages, (max_num_pages, 2, 16, 32, 128), "float16") + page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) + nnz_pages = T.int32(is_size_var=True) + page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) + length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) + k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) + q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) + output = T.match_buffer(var_output, (total_len, 16, 128), "float16") + lse = T.match_buffer(var_lse, (total_len, 16)) + tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) + total_tree_order_len = T.int32(is_size_var=True) + tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) + # with T.block("root"): + assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." + for lbx in T.thread_binding(16, thread="blockIdx.x"): + for lby in T.thread_binding(16, thread="blockIdx.y"): + for lty in T.thread_binding(4, thread="threadIdx.y"): + for ltx in T.thread_binding(32, thread="threadIdx.x"): + with T.block("attn"): + bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) + T.reads() + T.writes() + tile_id = T.alloc_buffer((1,), "int32", scope="local") + batch_idx = T.alloc_buffer((1,), "int32", scope="local") + batch_tiles = T.alloc_buffer((1,), "int32", scope="local") + batch_rows = T.alloc_buffer((1,), "int32", scope="local") + iterator = T.alloc_buffer((1,), "int32", scope="local") + kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") + Q_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + K_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + V_smem = T.alloc_buffer((32, 128), "float16", scope="shared") + S_smem = T.alloc_buffer((32, 32), scope="shared") + S_local = T.alloc_buffer((32, 32), scope="local") + O_local = T.alloc_buffer((32, 128), scope="local") + m_smem = T.alloc_buffer((32,), scope="shared") + m_prev_smem = T.alloc_buffer((32,), scope="shared") + d_smem = T.alloc_buffer((32,), scope="shared") + m_new = T.alloc_buffer((1,), scope="local") + m_prev = T.alloc_buffer((1,), scope="local") + d_new = T.alloc_buffer((1,), scope="local") + tile_id[0] = bx + batch_idx[0] = 0 + batch_rows[0] = q_indptr[1] - q_indptr[0] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + while T.tvm_thread_invariant(batch_idx[0] < batch_size): + while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: + tile_id[0] = tile_id[0] - batch_tiles[0] + batch_idx[0] = batch_idx[0] + 1 + if batch_idx[0] < batch_size: + b_idx: T.int32 = batch_idx[0] + batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] + batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 + if T.tvm_thread_invariant(batch_idx[0] < batch_size): + b_idx: T.int32 = batch_idx[0] + LH_start: T.int32 = tile_id[0] * 32 + q_indptr_val: T.int32 = q_indptr[b_idx] + cur_page_indptr_begin: T.int32 = page_indptr[b_idx] + cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] + kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + m_smem[row] = T.float32(-50000.0) + d_smem[row] = T.float32(1.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = T.float32(0.0) + T.tvm_storage_sync("shared") + for li_lj_fused_0 in range(8): + for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for li_lj_fused_3 in T.vectorized(4): + with T.block("Q_load"): + i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 128) + j = T.axis.spatial(128, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = q_indptr_val + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + freq = T.float32() + Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 64, q[cur_L, cur_H_qo, j + 64] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 64]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 128) / T.float32(128.0))}), q[cur_L, cur_H_qo, j]) + else: + Q_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for iterator_1 in range((kv_chunk_len[0] + 31) // 32): + L_kv_start: T.int32 = iterator_1 * 32 + for lz_ly_fused_0 in range(8): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("K_load"): + i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + seq_offset: T.int32 = cur_L + page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] + page_offset: T.int32 = seq_offset % 32 + K_smem[i, j] = pages[page_no, 0, by, page_offset, j] + else: + K_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + for lz_ly_fused_0 in range(8): + for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): + for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): + for lz_ly_fused_3 in T.vectorized(4): + with T.block("V_load"): + i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 128) + j = T.axis.spatial(128, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 128) + T.reads() + T.writes() + cur_L: T.int32 = L_kv_start + i + if cur_L < kv_chunk_len[0]: + seq_offset: T.int32 = cur_L + page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] + page_offset: T.int32 = seq_offset % 32 + V_smem[i, j] = pages[page_no, 1, by, page_offset, j] + else: + V_smem[i, j] = T.float16(0.0) + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(Q_smem[0:32, 0:128], K_smem[0:32, 0:128]) + T.writes(S_local[0:32, 0:32]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(2, 4): + with T.block("S_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) + j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) + T.reads() + T.writes(S_local[i, j]) + S_local[i, j] = T.float32(0.0) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, li_1, lj_1, lk_1 in T.grid(16, 2, 4, 8): + with T.block("S_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) + k = T.axis.reduce(128, lk_0 * 8 + lk_1) + T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) + T.writes(S_local[i, j]) + S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.12751743082459868) + T.tvm_storage_sync("shared") + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(2, 4): + with T.block("S_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) + j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) + T.reads(S_local[i, j]) + T.writes(S_smem[i, j]) + S_smem[i, j] = S_local[i, j] + T.tvm_storage_sync("shared") + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update1"): + T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) + T.writes(m_prev[i], m_new[i], d_new[i]) + m_prev[i] = m_smem[row] + m_new[i] = m_smem[row] + row_: T.int32 = LH_start + row + for j in range(32): + if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): + m_new[i] = T.max(m_new[i], S_smem[row, j]) + d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + with T.block("update"): + T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) + T.writes(S_smem[row, 0:32]) + for j in range(32): + if row < 32: + row_: T.int32 = LH_start + row + if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): + S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) + else: + S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) + for i in range(1): + row: T.int32 = i * 32 * 4 + ty * 32 + tx + if row < 32: + with T.block("update"): + T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) + T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) + for j in range(32): + d_new[i] = d_new[i] + S_smem[row, j] + m_smem[row] = m_new[i] + d_smem[row] = d_new[i] + m_prev_smem[row] = m_prev[i] + T.tvm_storage_sync("shared") + with T.block(""): + T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:128]) + T.writes(O_local[0:32, 0:128]) + for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): + for li_1_init, lj_1_init in T.grid(4, 8): + with T.block("O_gemm_init"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) + j = T.axis.spatial(128, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 8 + lj_1_init) + T.reads() + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): + with T.block("O_gemm_update"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + k = T.axis.reduce(32, lk_0 * 8 + lk_1) + T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) + T.writes(O_local[i, j]) + O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) + for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): + for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): + for li_1, lj_1 in T.grid(4, 8): + with T.block("O_store"): + i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) + j = T.axis.spatial(128, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 8 + lj_1) + T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) + T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) + for li_0 in range(1): + for li_1 in T.thread_binding(4, thread="threadIdx.y"): + for li_2 in T.thread_binding(32, thread="threadIdx.x"): + with T.block("lse_store"): + i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) + T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) + T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) + T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) + cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) + cur_H_qo: T.int32 = by + if cur_L < q_indptr[b_idx + 1]: + lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) + tile_id[0] = tile_id[0] + 16 + + @R.function + def alloc_embedding_tensor() -> R.Tensor((2048, 2048), dtype="float16"): + R.func_attr({"relax.memory_plan_dynamic_func_output": True}) + gv: R.Tensor((2048, 2048), dtype="float16") = R.builtin.alloc_tensor(R.shape([2048, 2048]), R.dtype("float16"), R.prim_value(0), R.str("global")) + return gv + + @R.function + def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor(("vocab_size", 2048), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): + batch_size = T.int64() + vocab_size = T.int64() + R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + with R.dataflow(): + gpt_neox_embed_in_weight4: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] + gpt_neox_layers_0_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[1] + gpt_neox_layers_0_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[2] + gpt_neox_layers_0_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[3] + gpt_neox_layers_0_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[4] + gpt_neox_layers_0_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[5] + gpt_neox_layers_0_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[6] + gpt_neox_layers_0_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[7] + gpt_neox_layers_0_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[10] + gpt_neox_layers_0_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[11] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[12] + gpt_neox_layers_1_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[13] + gpt_neox_layers_1_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[14] + gpt_neox_layers_1_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[15] + gpt_neox_layers_1_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[16] + gpt_neox_layers_1_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[17] + gpt_neox_layers_1_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[18] + gpt_neox_layers_1_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[19] + gpt_neox_layers_1_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[20] + gpt_neox_layers_1_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[21] + gpt_neox_layers_1_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[22] + gpt_neox_layers_1_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[23] + gpt_neox_layers_1_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[24] + gpt_neox_layers_2_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[25] + gpt_neox_layers_2_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[26] + gpt_neox_layers_2_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[27] + gpt_neox_layers_2_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[28] + gpt_neox_layers_2_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[29] + gpt_neox_layers_2_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[30] + gpt_neox_layers_2_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[31] + gpt_neox_layers_2_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[32] + gpt_neox_layers_2_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[33] + gpt_neox_layers_2_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[34] + gpt_neox_layers_2_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[35] + gpt_neox_layers_2_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[36] + gpt_neox_layers_3_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[37] + gpt_neox_layers_3_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[38] + gpt_neox_layers_3_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[39] + gpt_neox_layers_3_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[40] + gpt_neox_layers_3_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[41] + gpt_neox_layers_3_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[42] + gpt_neox_layers_3_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[43] + gpt_neox_layers_3_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[44] + gpt_neox_layers_3_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[45] + gpt_neox_layers_3_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[46] + gpt_neox_layers_3_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[47] + gpt_neox_layers_3_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[48] + gpt_neox_layers_4_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[49] + gpt_neox_layers_4_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[50] + gpt_neox_layers_4_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[51] + gpt_neox_layers_4_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[52] + gpt_neox_layers_4_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[53] + gpt_neox_layers_4_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[54] + gpt_neox_layers_4_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[55] + gpt_neox_layers_4_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[56] + gpt_neox_layers_4_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[57] + gpt_neox_layers_4_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[58] + gpt_neox_layers_4_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[59] + gpt_neox_layers_4_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[60] + gpt_neox_layers_5_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[61] + gpt_neox_layers_5_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[62] + gpt_neox_layers_5_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[63] + gpt_neox_layers_5_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[64] + gpt_neox_layers_5_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[65] + gpt_neox_layers_5_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[66] + gpt_neox_layers_5_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[67] + gpt_neox_layers_5_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[68] + gpt_neox_layers_5_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[69] + gpt_neox_layers_5_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[70] + gpt_neox_layers_5_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[71] + gpt_neox_layers_5_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[72] + gpt_neox_layers_6_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[73] + gpt_neox_layers_6_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[74] + gpt_neox_layers_6_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[75] + gpt_neox_layers_6_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[76] + gpt_neox_layers_6_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[77] + gpt_neox_layers_6_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[78] + gpt_neox_layers_6_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[79] + gpt_neox_layers_6_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[80] + gpt_neox_layers_6_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[81] + gpt_neox_layers_6_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[82] + gpt_neox_layers_6_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[83] + gpt_neox_layers_6_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[84] + gpt_neox_layers_7_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[85] + gpt_neox_layers_7_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[86] + gpt_neox_layers_7_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[87] + gpt_neox_layers_7_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[88] + gpt_neox_layers_7_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[89] + gpt_neox_layers_7_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[90] + gpt_neox_layers_7_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[91] + gpt_neox_layers_7_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[92] + gpt_neox_layers_7_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[93] + gpt_neox_layers_7_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[94] + gpt_neox_layers_7_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[95] + gpt_neox_layers_7_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[96] + gpt_neox_layers_8_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[97] + gpt_neox_layers_8_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[98] + gpt_neox_layers_8_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[99] + gpt_neox_layers_8_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[100] + gpt_neox_layers_8_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[101] + gpt_neox_layers_8_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[102] + gpt_neox_layers_8_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[103] + gpt_neox_layers_8_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[104] + gpt_neox_layers_8_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[105] + gpt_neox_layers_8_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[106] + gpt_neox_layers_8_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[107] + gpt_neox_layers_8_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[108] + gpt_neox_layers_9_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[109] + gpt_neox_layers_9_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[110] + gpt_neox_layers_9_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[111] + gpt_neox_layers_9_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[112] + gpt_neox_layers_9_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[113] + gpt_neox_layers_9_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[114] + gpt_neox_layers_9_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[115] + gpt_neox_layers_9_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[116] + gpt_neox_layers_9_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[117] + gpt_neox_layers_9_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[118] + gpt_neox_layers_9_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[119] + gpt_neox_layers_9_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[120] + gpt_neox_layers_10_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[121] + gpt_neox_layers_10_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[122] + gpt_neox_layers_10_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[123] + gpt_neox_layers_10_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[124] + gpt_neox_layers_10_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[125] + gpt_neox_layers_10_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[126] + gpt_neox_layers_10_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[127] + gpt_neox_layers_10_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[128] + gpt_neox_layers_10_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[129] + gpt_neox_layers_10_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[130] + gpt_neox_layers_10_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[131] + gpt_neox_layers_10_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[132] + gpt_neox_layers_11_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[133] + gpt_neox_layers_11_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[134] + gpt_neox_layers_11_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[135] + gpt_neox_layers_11_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[136] + gpt_neox_layers_11_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[137] + gpt_neox_layers_11_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[138] + gpt_neox_layers_11_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[139] + gpt_neox_layers_11_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[140] + gpt_neox_layers_11_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[141] + gpt_neox_layers_11_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[142] + gpt_neox_layers_11_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[143] + gpt_neox_layers_11_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[144] + gpt_neox_layers_12_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[145] + gpt_neox_layers_12_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[146] + gpt_neox_layers_12_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[147] + gpt_neox_layers_12_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[148] + gpt_neox_layers_12_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[149] + gpt_neox_layers_12_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[150] + gpt_neox_layers_12_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[151] + gpt_neox_layers_12_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[152] + gpt_neox_layers_12_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[153] + gpt_neox_layers_12_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[154] + gpt_neox_layers_12_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[155] + gpt_neox_layers_12_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[156] + gpt_neox_layers_13_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[157] + gpt_neox_layers_13_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[158] + gpt_neox_layers_13_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[159] + gpt_neox_layers_13_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[160] + gpt_neox_layers_13_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[161] + gpt_neox_layers_13_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[162] + gpt_neox_layers_13_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[163] + gpt_neox_layers_13_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[164] + gpt_neox_layers_13_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[165] + gpt_neox_layers_13_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[166] + gpt_neox_layers_13_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[167] + gpt_neox_layers_13_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[168] + gpt_neox_layers_14_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[169] + gpt_neox_layers_14_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[170] + gpt_neox_layers_14_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[171] + gpt_neox_layers_14_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[172] + gpt_neox_layers_14_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[173] + gpt_neox_layers_14_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[174] + gpt_neox_layers_14_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[175] + gpt_neox_layers_14_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[176] + gpt_neox_layers_14_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[177] + gpt_neox_layers_14_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[178] + gpt_neox_layers_14_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[179] + gpt_neox_layers_14_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[180] + gpt_neox_layers_15_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[181] + gpt_neox_layers_15_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[182] + gpt_neox_layers_15_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[183] + gpt_neox_layers_15_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[184] + gpt_neox_layers_15_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[185] + gpt_neox_layers_15_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[186] + gpt_neox_layers_15_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[187] + gpt_neox_layers_15_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[188] + gpt_neox_layers_15_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[189] + gpt_neox_layers_15_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[190] + gpt_neox_layers_15_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[191] + gpt_neox_layers_15_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[192] + gpt_neox_layers_16_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[193] + gpt_neox_layers_16_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[194] + gpt_neox_layers_16_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[195] + gpt_neox_layers_16_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[196] + gpt_neox_layers_16_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[197] + gpt_neox_layers_16_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[198] + gpt_neox_layers_16_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[199] + gpt_neox_layers_16_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[200] + gpt_neox_layers_16_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[201] + gpt_neox_layers_16_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[202] + gpt_neox_layers_16_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[203] + gpt_neox_layers_16_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[204] + gpt_neox_layers_17_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[205] + gpt_neox_layers_17_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[206] + gpt_neox_layers_17_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[207] + gpt_neox_layers_17_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[208] + gpt_neox_layers_17_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[209] + gpt_neox_layers_17_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[210] + gpt_neox_layers_17_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[211] + gpt_neox_layers_17_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[212] + gpt_neox_layers_17_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[213] + gpt_neox_layers_17_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[214] + gpt_neox_layers_17_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[215] + gpt_neox_layers_17_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[216] + gpt_neox_layers_18_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[217] + gpt_neox_layers_18_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[218] + gpt_neox_layers_18_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[219] + gpt_neox_layers_18_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[220] + gpt_neox_layers_18_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[221] + gpt_neox_layers_18_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[222] + gpt_neox_layers_18_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[223] + gpt_neox_layers_18_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[224] + gpt_neox_layers_18_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[225] + gpt_neox_layers_18_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[226] + gpt_neox_layers_18_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[227] + gpt_neox_layers_18_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[228] + gpt_neox_layers_19_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[229] + gpt_neox_layers_19_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[230] + gpt_neox_layers_19_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[231] + gpt_neox_layers_19_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[232] + gpt_neox_layers_19_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[233] + gpt_neox_layers_19_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[234] + gpt_neox_layers_19_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[235] + gpt_neox_layers_19_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[236] + gpt_neox_layers_19_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[237] + gpt_neox_layers_19_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[238] + gpt_neox_layers_19_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[239] + gpt_neox_layers_19_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[240] + gpt_neox_layers_20_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[241] + gpt_neox_layers_20_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[242] + gpt_neox_layers_20_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[243] + gpt_neox_layers_20_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[244] + gpt_neox_layers_20_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[245] + gpt_neox_layers_20_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[246] + gpt_neox_layers_20_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[247] + gpt_neox_layers_20_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[248] + gpt_neox_layers_20_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[249] + gpt_neox_layers_20_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[250] + gpt_neox_layers_20_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[251] + gpt_neox_layers_20_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[252] + gpt_neox_layers_21_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[253] + gpt_neox_layers_21_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[254] + gpt_neox_layers_21_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[255] + gpt_neox_layers_21_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[256] + gpt_neox_layers_21_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[257] + gpt_neox_layers_21_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[258] + gpt_neox_layers_21_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[259] + gpt_neox_layers_21_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[260] + gpt_neox_layers_21_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[261] + gpt_neox_layers_21_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[262] + gpt_neox_layers_21_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[263] + gpt_neox_layers_21_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[264] + gpt_neox_layers_22_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[265] + gpt_neox_layers_22_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[266] + gpt_neox_layers_22_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[267] + gpt_neox_layers_22_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[268] + gpt_neox_layers_22_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[269] + gpt_neox_layers_22_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[270] + gpt_neox_layers_22_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[271] + gpt_neox_layers_22_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[272] + gpt_neox_layers_22_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[273] + gpt_neox_layers_22_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[274] + gpt_neox_layers_22_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[275] + gpt_neox_layers_22_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[276] + gpt_neox_layers_23_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[277] + gpt_neox_layers_23_input_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[278] + gpt_neox_layers_23_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[279] + gpt_neox_layers_23_post_attention_layernorm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[280] + gpt_neox_layers_23_attention_query_key_value_weight4: R.Tensor((6144, 2048), dtype="float16") = packed_params[281] + gpt_neox_layers_23_attention_query_key_value_bias4: R.Tensor((6144,), dtype="float16") = packed_params[282] + gpt_neox_layers_23_attention_dense_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[283] + gpt_neox_layers_23_attention_dense_bias4: R.Tensor((2048,), dtype="float16") = packed_params[284] + gpt_neox_layers_23_mlp_dense_h_to_4h_weight4: R.Tensor((8192, 2048), dtype="float16") = packed_params[285] + gpt_neox_layers_23_mlp_dense_h_to_4h_bias4: R.Tensor((8192,), dtype="float32") = packed_params[286] + gpt_neox_layers_23_mlp_dense_4h_to_h_weight4: R.Tensor((2048, 8192), dtype="float16") = packed_params[287] + gpt_neox_layers_23_mlp_dense_4h_to_h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[288] + gpt_neox_final_layer_norm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[289] + gpt_neox_final_layer_norm_bias4: R.Tensor((2048,), dtype="float16") = packed_params[290] + embed_out_weight4: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[291] + layer_norm147: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight4, gpt_neox_layers_0_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims291: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_query_key_value_weight4, axes=None) + matmul291: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm147, permute_dims291, out_dtype="void") + add432: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul291, gpt_neox_layers_0_attention_query_key_value_bias4) + reshape288: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add432, R.shape([batch_size, 1, 48, 128])) + reshape289: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape288, R.shape([batch_size, 48, 128])) + lv73 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape289), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape290: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv73, R.shape([batch_size, 1, 16, 128])) + reshape291: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape290, R.shape([batch_size, 1, 2048])) + permute_dims292: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_dense_weight4, axes=None) + matmul292: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape291, permute_dims292, out_dtype="void") + add433: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul292, gpt_neox_layers_0_attention_dense_bias4) + layer_norm148: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight4, gpt_neox_layers_0_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims293: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, axes=None) + matmul293: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm148, permute_dims293, out_dtype="float32") + add434: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul293, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4) + gelu72: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add434) + astype147: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu72, dtype="float16") + permute_dims294: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, axes=None) + matmul294: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype147, permute_dims294, out_dtype="float32") + add435: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul294, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4) + astype148: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add435, dtype="float16") + add436: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype148, add433) + add437: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add436, input_embeds) + layer_norm149: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add437, gpt_neox_layers_1_input_layernorm_weight4, gpt_neox_layers_1_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims295: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_query_key_value_weight4, axes=None) + matmul295: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm149, permute_dims295, out_dtype="void") + add438: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul295, gpt_neox_layers_1_attention_query_key_value_bias4) + reshape292: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add438, R.shape([batch_size, 1, 48, 128])) + reshape293: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape292, R.shape([batch_size, 48, 128])) + lv74 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape293), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape294: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv74, R.shape([batch_size, 1, 16, 128])) + reshape295: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape294, R.shape([batch_size, 1, 2048])) + permute_dims296: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_dense_weight4, axes=None) + matmul296: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape295, permute_dims296, out_dtype="void") + add439: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul296, gpt_neox_layers_1_attention_dense_bias4) + layer_norm150: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add437, gpt_neox_layers_1_post_attention_layernorm_weight4, gpt_neox_layers_1_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims297: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_h_to_4h_weight4, axes=None) + matmul297: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm150, permute_dims297, out_dtype="float32") + add440: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul297, gpt_neox_layers_1_mlp_dense_h_to_4h_bias4) + gelu73: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add440) + astype149: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu73, dtype="float16") + permute_dims298: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_4h_to_h_weight4, axes=None) + matmul298: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype149, permute_dims298, out_dtype="float32") + add441: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul298, gpt_neox_layers_1_mlp_dense_4h_to_h_bias4) + astype150: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add441, dtype="float16") + add442: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype150, add439) + add443: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add442, add437) + layer_norm151: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add443, gpt_neox_layers_2_input_layernorm_weight4, gpt_neox_layers_2_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims299: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_query_key_value_weight4, axes=None) + matmul299: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm151, permute_dims299, out_dtype="void") + add444: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul299, gpt_neox_layers_2_attention_query_key_value_bias4) + reshape296: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add444, R.shape([batch_size, 1, 48, 128])) + reshape297: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape296, R.shape([batch_size, 48, 128])) + lv75 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape297), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape298: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv75, R.shape([batch_size, 1, 16, 128])) + reshape299: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape298, R.shape([batch_size, 1, 2048])) + permute_dims300: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_dense_weight4, axes=None) + matmul300: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape299, permute_dims300, out_dtype="void") + add445: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul300, gpt_neox_layers_2_attention_dense_bias4) + layer_norm152: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add443, gpt_neox_layers_2_post_attention_layernorm_weight4, gpt_neox_layers_2_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims301: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_h_to_4h_weight4, axes=None) + matmul301: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm152, permute_dims301, out_dtype="float32") + add446: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul301, gpt_neox_layers_2_mlp_dense_h_to_4h_bias4) + gelu74: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add446) + astype151: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu74, dtype="float16") + permute_dims302: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_4h_to_h_weight4, axes=None) + matmul302: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype151, permute_dims302, out_dtype="float32") + add447: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul302, gpt_neox_layers_2_mlp_dense_4h_to_h_bias4) + astype152: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add447, dtype="float16") + add448: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype152, add445) + add449: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add448, add443) + layer_norm153: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add449, gpt_neox_layers_3_input_layernorm_weight4, gpt_neox_layers_3_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims303: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_query_key_value_weight4, axes=None) + matmul303: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm153, permute_dims303, out_dtype="void") + add450: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul303, gpt_neox_layers_3_attention_query_key_value_bias4) + reshape300: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add450, R.shape([batch_size, 1, 48, 128])) + reshape301: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape300, R.shape([batch_size, 48, 128])) + lv76 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape301), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape302: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv76, R.shape([batch_size, 1, 16, 128])) + reshape303: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape302, R.shape([batch_size, 1, 2048])) + permute_dims304: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_dense_weight4, axes=None) + matmul304: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape303, permute_dims304, out_dtype="void") + add451: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul304, gpt_neox_layers_3_attention_dense_bias4) + layer_norm154: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add449, gpt_neox_layers_3_post_attention_layernorm_weight4, gpt_neox_layers_3_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims305: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_h_to_4h_weight4, axes=None) + matmul305: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm154, permute_dims305, out_dtype="float32") + add452: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul305, gpt_neox_layers_3_mlp_dense_h_to_4h_bias4) + gelu75: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add452) + astype153: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu75, dtype="float16") + permute_dims306: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_4h_to_h_weight4, axes=None) + matmul306: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype153, permute_dims306, out_dtype="float32") + add453: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul306, gpt_neox_layers_3_mlp_dense_4h_to_h_bias4) + astype154: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add453, dtype="float16") + add454: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype154, add451) + add455: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add454, add449) + layer_norm155: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add455, gpt_neox_layers_4_input_layernorm_weight4, gpt_neox_layers_4_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims307: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_query_key_value_weight4, axes=None) + matmul307: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm155, permute_dims307, out_dtype="void") + add456: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul307, gpt_neox_layers_4_attention_query_key_value_bias4) + reshape304: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add456, R.shape([batch_size, 1, 48, 128])) + reshape305: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape304, R.shape([batch_size, 48, 128])) + lv77 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape305), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape306: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv77, R.shape([batch_size, 1, 16, 128])) + reshape307: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape306, R.shape([batch_size, 1, 2048])) + permute_dims308: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_dense_weight4, axes=None) + matmul308: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape307, permute_dims308, out_dtype="void") + add457: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul308, gpt_neox_layers_4_attention_dense_bias4) + layer_norm156: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add455, gpt_neox_layers_4_post_attention_layernorm_weight4, gpt_neox_layers_4_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims309: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_h_to_4h_weight4, axes=None) + matmul309: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm156, permute_dims309, out_dtype="float32") + add458: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul309, gpt_neox_layers_4_mlp_dense_h_to_4h_bias4) + gelu76: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add458) + astype155: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu76, dtype="float16") + permute_dims310: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_4h_to_h_weight4, axes=None) + matmul310: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype155, permute_dims310, out_dtype="float32") + add459: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul310, gpt_neox_layers_4_mlp_dense_4h_to_h_bias4) + astype156: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add459, dtype="float16") + add460: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype156, add457) + add461: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add460, add455) + layer_norm157: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add461, gpt_neox_layers_5_input_layernorm_weight4, gpt_neox_layers_5_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims311: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_query_key_value_weight4, axes=None) + matmul311: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm157, permute_dims311, out_dtype="void") + add462: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul311, gpt_neox_layers_5_attention_query_key_value_bias4) + reshape308: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add462, R.shape([batch_size, 1, 48, 128])) + reshape309: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape308, R.shape([batch_size, 48, 128])) + lv78 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape309), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape310: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv78, R.shape([batch_size, 1, 16, 128])) + reshape311: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape310, R.shape([batch_size, 1, 2048])) + permute_dims312: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_dense_weight4, axes=None) + matmul312: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape311, permute_dims312, out_dtype="void") + add463: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul312, gpt_neox_layers_5_attention_dense_bias4) + layer_norm158: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add461, gpt_neox_layers_5_post_attention_layernorm_weight4, gpt_neox_layers_5_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims313: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_h_to_4h_weight4, axes=None) + matmul313: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm158, permute_dims313, out_dtype="float32") + add464: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul313, gpt_neox_layers_5_mlp_dense_h_to_4h_bias4) + gelu77: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add464) + astype157: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu77, dtype="float16") + permute_dims314: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_4h_to_h_weight4, axes=None) + matmul314: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype157, permute_dims314, out_dtype="float32") + add465: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul314, gpt_neox_layers_5_mlp_dense_4h_to_h_bias4) + astype158: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add465, dtype="float16") + add466: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype158, add463) + add467: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add466, add461) + layer_norm159: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add467, gpt_neox_layers_6_input_layernorm_weight4, gpt_neox_layers_6_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims315: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_query_key_value_weight4, axes=None) + matmul315: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm159, permute_dims315, out_dtype="void") + add468: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul315, gpt_neox_layers_6_attention_query_key_value_bias4) + reshape312: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add468, R.shape([batch_size, 1, 48, 128])) + reshape313: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape312, R.shape([batch_size, 48, 128])) + lv79 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape313), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape314: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv79, R.shape([batch_size, 1, 16, 128])) + reshape315: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape314, R.shape([batch_size, 1, 2048])) + permute_dims316: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_dense_weight4, axes=None) + matmul316: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape315, permute_dims316, out_dtype="void") + add469: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul316, gpt_neox_layers_6_attention_dense_bias4) + layer_norm160: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add467, gpt_neox_layers_6_post_attention_layernorm_weight4, gpt_neox_layers_6_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims317: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_h_to_4h_weight4, axes=None) + matmul317: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm160, permute_dims317, out_dtype="float32") + add470: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul317, gpt_neox_layers_6_mlp_dense_h_to_4h_bias4) + gelu78: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add470) + astype159: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu78, dtype="float16") + permute_dims318: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_4h_to_h_weight4, axes=None) + matmul318: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype159, permute_dims318, out_dtype="float32") + add471: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul318, gpt_neox_layers_6_mlp_dense_4h_to_h_bias4) + astype160: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add471, dtype="float16") + add472: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype160, add469) + add473: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add472, add467) + layer_norm161: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add473, gpt_neox_layers_7_input_layernorm_weight4, gpt_neox_layers_7_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims319: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_query_key_value_weight4, axes=None) + matmul319: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm161, permute_dims319, out_dtype="void") + add474: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul319, gpt_neox_layers_7_attention_query_key_value_bias4) + reshape316: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add474, R.shape([batch_size, 1, 48, 128])) + reshape317: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape316, R.shape([batch_size, 48, 128])) + lv80 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape317), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape318: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv80, R.shape([batch_size, 1, 16, 128])) + reshape319: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape318, R.shape([batch_size, 1, 2048])) + permute_dims320: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_dense_weight4, axes=None) + matmul320: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape319, permute_dims320, out_dtype="void") + add475: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul320, gpt_neox_layers_7_attention_dense_bias4) + layer_norm162: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add473, gpt_neox_layers_7_post_attention_layernorm_weight4, gpt_neox_layers_7_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims321: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_h_to_4h_weight4, axes=None) + matmul321: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm162, permute_dims321, out_dtype="float32") + add476: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul321, gpt_neox_layers_7_mlp_dense_h_to_4h_bias4) + gelu79: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add476) + astype161: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu79, dtype="float16") + permute_dims322: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_4h_to_h_weight4, axes=None) + matmul322: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype161, permute_dims322, out_dtype="float32") + add477: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul322, gpt_neox_layers_7_mlp_dense_4h_to_h_bias4) + astype162: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add477, dtype="float16") + add478: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype162, add475) + add479: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add478, add473) + layer_norm163: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add479, gpt_neox_layers_8_input_layernorm_weight4, gpt_neox_layers_8_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims323: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_query_key_value_weight4, axes=None) + matmul323: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm163, permute_dims323, out_dtype="void") + add480: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul323, gpt_neox_layers_8_attention_query_key_value_bias4) + reshape320: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add480, R.shape([batch_size, 1, 48, 128])) + reshape321: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape320, R.shape([batch_size, 48, 128])) + lv81 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape321), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape322: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv81, R.shape([batch_size, 1, 16, 128])) + reshape323: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape322, R.shape([batch_size, 1, 2048])) + permute_dims324: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_dense_weight4, axes=None) + matmul324: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape323, permute_dims324, out_dtype="void") + add481: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul324, gpt_neox_layers_8_attention_dense_bias4) + layer_norm164: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add479, gpt_neox_layers_8_post_attention_layernorm_weight4, gpt_neox_layers_8_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims325: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_h_to_4h_weight4, axes=None) + matmul325: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm164, permute_dims325, out_dtype="float32") + add482: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul325, gpt_neox_layers_8_mlp_dense_h_to_4h_bias4) + gelu80: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add482) + astype163: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu80, dtype="float16") + permute_dims326: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_4h_to_h_weight4, axes=None) + matmul326: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype163, permute_dims326, out_dtype="float32") + add483: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul326, gpt_neox_layers_8_mlp_dense_4h_to_h_bias4) + astype164: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add483, dtype="float16") + add484: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype164, add481) + add485: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add484, add479) + layer_norm165: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add485, gpt_neox_layers_9_input_layernorm_weight4, gpt_neox_layers_9_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims327: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_query_key_value_weight4, axes=None) + matmul327: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm165, permute_dims327, out_dtype="void") + add486: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul327, gpt_neox_layers_9_attention_query_key_value_bias4) + reshape324: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add486, R.shape([batch_size, 1, 48, 128])) + reshape325: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape324, R.shape([batch_size, 48, 128])) + lv82 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape325), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape326: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv82, R.shape([batch_size, 1, 16, 128])) + reshape327: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape326, R.shape([batch_size, 1, 2048])) + permute_dims328: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_dense_weight4, axes=None) + matmul328: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape327, permute_dims328, out_dtype="void") + add487: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul328, gpt_neox_layers_9_attention_dense_bias4) + layer_norm166: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add485, gpt_neox_layers_9_post_attention_layernorm_weight4, gpt_neox_layers_9_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims329: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_h_to_4h_weight4, axes=None) + matmul329: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm166, permute_dims329, out_dtype="float32") + add488: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul329, gpt_neox_layers_9_mlp_dense_h_to_4h_bias4) + gelu81: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add488) + astype165: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu81, dtype="float16") + permute_dims330: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_4h_to_h_weight4, axes=None) + matmul330: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype165, permute_dims330, out_dtype="float32") + add489: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul330, gpt_neox_layers_9_mlp_dense_4h_to_h_bias4) + astype166: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add489, dtype="float16") + add490: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype166, add487) + add491: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add490, add485) + layer_norm167: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add491, gpt_neox_layers_10_input_layernorm_weight4, gpt_neox_layers_10_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims331: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_query_key_value_weight4, axes=None) + matmul331: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm167, permute_dims331, out_dtype="void") + add492: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul331, gpt_neox_layers_10_attention_query_key_value_bias4) + reshape328: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add492, R.shape([batch_size, 1, 48, 128])) + reshape329: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape328, R.shape([batch_size, 48, 128])) + lv83 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape329), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape330: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv83, R.shape([batch_size, 1, 16, 128])) + reshape331: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape330, R.shape([batch_size, 1, 2048])) + permute_dims332: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_dense_weight4, axes=None) + matmul332: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape331, permute_dims332, out_dtype="void") + add493: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul332, gpt_neox_layers_10_attention_dense_bias4) + layer_norm168: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add491, gpt_neox_layers_10_post_attention_layernorm_weight4, gpt_neox_layers_10_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims333: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_h_to_4h_weight4, axes=None) + matmul333: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm168, permute_dims333, out_dtype="float32") + add494: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul333, gpt_neox_layers_10_mlp_dense_h_to_4h_bias4) + gelu82: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add494) + astype167: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu82, dtype="float16") + permute_dims334: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_4h_to_h_weight4, axes=None) + matmul334: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype167, permute_dims334, out_dtype="float32") + add495: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul334, gpt_neox_layers_10_mlp_dense_4h_to_h_bias4) + astype168: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add495, dtype="float16") + add496: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype168, add493) + add497: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add496, add491) + layer_norm169: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add497, gpt_neox_layers_11_input_layernorm_weight4, gpt_neox_layers_11_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims335: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_query_key_value_weight4, axes=None) + matmul335: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm169, permute_dims335, out_dtype="void") + add498: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul335, gpt_neox_layers_11_attention_query_key_value_bias4) + reshape332: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add498, R.shape([batch_size, 1, 48, 128])) + reshape333: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape332, R.shape([batch_size, 48, 128])) + lv84 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape333), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape334: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv84, R.shape([batch_size, 1, 16, 128])) + reshape335: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape334, R.shape([batch_size, 1, 2048])) + permute_dims336: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_dense_weight4, axes=None) + matmul336: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape335, permute_dims336, out_dtype="void") + add499: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul336, gpt_neox_layers_11_attention_dense_bias4) + layer_norm170: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add497, gpt_neox_layers_11_post_attention_layernorm_weight4, gpt_neox_layers_11_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims337: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_h_to_4h_weight4, axes=None) + matmul337: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm170, permute_dims337, out_dtype="float32") + add500: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul337, gpt_neox_layers_11_mlp_dense_h_to_4h_bias4) + gelu83: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add500) + astype169: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu83, dtype="float16") + permute_dims338: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_4h_to_h_weight4, axes=None) + matmul338: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype169, permute_dims338, out_dtype="float32") + add501: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul338, gpt_neox_layers_11_mlp_dense_4h_to_h_bias4) + astype170: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add501, dtype="float16") + add502: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype170, add499) + add503: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add502, add497) + layer_norm171: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add503, gpt_neox_layers_12_input_layernorm_weight4, gpt_neox_layers_12_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims339: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_query_key_value_weight4, axes=None) + matmul339: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm171, permute_dims339, out_dtype="void") + add504: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul339, gpt_neox_layers_12_attention_query_key_value_bias4) + reshape336: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add504, R.shape([batch_size, 1, 48, 128])) + reshape337: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape336, R.shape([batch_size, 48, 128])) + lv85 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape337), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape338: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv85, R.shape([batch_size, 1, 16, 128])) + reshape339: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape338, R.shape([batch_size, 1, 2048])) + permute_dims340: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_dense_weight4, axes=None) + matmul340: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape339, permute_dims340, out_dtype="void") + add505: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul340, gpt_neox_layers_12_attention_dense_bias4) + layer_norm172: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add503, gpt_neox_layers_12_post_attention_layernorm_weight4, gpt_neox_layers_12_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims341: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_h_to_4h_weight4, axes=None) + matmul341: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm172, permute_dims341, out_dtype="float32") + add506: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul341, gpt_neox_layers_12_mlp_dense_h_to_4h_bias4) + gelu84: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add506) + astype171: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu84, dtype="float16") + permute_dims342: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_4h_to_h_weight4, axes=None) + matmul342: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype171, permute_dims342, out_dtype="float32") + add507: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul342, gpt_neox_layers_12_mlp_dense_4h_to_h_bias4) + astype172: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add507, dtype="float16") + add508: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype172, add505) + add509: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add508, add503) + layer_norm173: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add509, gpt_neox_layers_13_input_layernorm_weight4, gpt_neox_layers_13_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims343: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_query_key_value_weight4, axes=None) + matmul343: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm173, permute_dims343, out_dtype="void") + add510: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul343, gpt_neox_layers_13_attention_query_key_value_bias4) + reshape340: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add510, R.shape([batch_size, 1, 48, 128])) + reshape341: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape340, R.shape([batch_size, 48, 128])) + lv86 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape341), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape342: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv86, R.shape([batch_size, 1, 16, 128])) + reshape343: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape342, R.shape([batch_size, 1, 2048])) + permute_dims344: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_dense_weight4, axes=None) + matmul344: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape343, permute_dims344, out_dtype="void") + add511: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul344, gpt_neox_layers_13_attention_dense_bias4) + layer_norm174: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add509, gpt_neox_layers_13_post_attention_layernorm_weight4, gpt_neox_layers_13_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims345: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_h_to_4h_weight4, axes=None) + matmul345: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm174, permute_dims345, out_dtype="float32") + add512: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul345, gpt_neox_layers_13_mlp_dense_h_to_4h_bias4) + gelu85: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add512) + astype173: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu85, dtype="float16") + permute_dims346: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_4h_to_h_weight4, axes=None) + matmul346: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype173, permute_dims346, out_dtype="float32") + add513: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul346, gpt_neox_layers_13_mlp_dense_4h_to_h_bias4) + astype174: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add513, dtype="float16") + add514: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype174, add511) + add515: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add514, add509) + layer_norm175: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add515, gpt_neox_layers_14_input_layernorm_weight4, gpt_neox_layers_14_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims347: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_query_key_value_weight4, axes=None) + matmul347: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm175, permute_dims347, out_dtype="void") + add516: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul347, gpt_neox_layers_14_attention_query_key_value_bias4) + reshape344: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add516, R.shape([batch_size, 1, 48, 128])) + reshape345: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape344, R.shape([batch_size, 48, 128])) + lv87 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape345), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape346: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv87, R.shape([batch_size, 1, 16, 128])) + reshape347: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape346, R.shape([batch_size, 1, 2048])) + permute_dims348: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_dense_weight4, axes=None) + matmul348: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape347, permute_dims348, out_dtype="void") + add517: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul348, gpt_neox_layers_14_attention_dense_bias4) + layer_norm176: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add515, gpt_neox_layers_14_post_attention_layernorm_weight4, gpt_neox_layers_14_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims349: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_h_to_4h_weight4, axes=None) + matmul349: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm176, permute_dims349, out_dtype="float32") + add518: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul349, gpt_neox_layers_14_mlp_dense_h_to_4h_bias4) + gelu86: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add518) + astype175: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu86, dtype="float16") + permute_dims350: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_4h_to_h_weight4, axes=None) + matmul350: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype175, permute_dims350, out_dtype="float32") + add519: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul350, gpt_neox_layers_14_mlp_dense_4h_to_h_bias4) + astype176: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add519, dtype="float16") + add520: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype176, add517) + add521: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add520, add515) + layer_norm177: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add521, gpt_neox_layers_15_input_layernorm_weight4, gpt_neox_layers_15_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims351: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_query_key_value_weight4, axes=None) + matmul351: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm177, permute_dims351, out_dtype="void") + add522: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul351, gpt_neox_layers_15_attention_query_key_value_bias4) + reshape348: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add522, R.shape([batch_size, 1, 48, 128])) + reshape349: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape348, R.shape([batch_size, 48, 128])) + lv88 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape349), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape350: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv88, R.shape([batch_size, 1, 16, 128])) + reshape351: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape350, R.shape([batch_size, 1, 2048])) + permute_dims352: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_dense_weight4, axes=None) + matmul352: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape351, permute_dims352, out_dtype="void") + add523: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul352, gpt_neox_layers_15_attention_dense_bias4) + layer_norm178: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add521, gpt_neox_layers_15_post_attention_layernorm_weight4, gpt_neox_layers_15_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims353: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_h_to_4h_weight4, axes=None) + matmul353: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm178, permute_dims353, out_dtype="float32") + add524: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul353, gpt_neox_layers_15_mlp_dense_h_to_4h_bias4) + gelu87: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add524) + astype177: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu87, dtype="float16") + permute_dims354: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_4h_to_h_weight4, axes=None) + matmul354: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype177, permute_dims354, out_dtype="float32") + add525: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul354, gpt_neox_layers_15_mlp_dense_4h_to_h_bias4) + astype178: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add525, dtype="float16") + add526: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype178, add523) + add527: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add526, add521) + layer_norm179: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add527, gpt_neox_layers_16_input_layernorm_weight4, gpt_neox_layers_16_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims355: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_query_key_value_weight4, axes=None) + matmul355: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm179, permute_dims355, out_dtype="void") + add528: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul355, gpt_neox_layers_16_attention_query_key_value_bias4) + reshape352: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add528, R.shape([batch_size, 1, 48, 128])) + reshape353: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape352, R.shape([batch_size, 48, 128])) + lv89 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape353), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape354: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv89, R.shape([batch_size, 1, 16, 128])) + reshape355: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape354, R.shape([batch_size, 1, 2048])) + permute_dims356: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_dense_weight4, axes=None) + matmul356: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape355, permute_dims356, out_dtype="void") + add529: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul356, gpt_neox_layers_16_attention_dense_bias4) + layer_norm180: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add527, gpt_neox_layers_16_post_attention_layernorm_weight4, gpt_neox_layers_16_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims357: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_h_to_4h_weight4, axes=None) + matmul357: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm180, permute_dims357, out_dtype="float32") + add530: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul357, gpt_neox_layers_16_mlp_dense_h_to_4h_bias4) + gelu88: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add530) + astype179: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu88, dtype="float16") + permute_dims358: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_4h_to_h_weight4, axes=None) + matmul358: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype179, permute_dims358, out_dtype="float32") + add531: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul358, gpt_neox_layers_16_mlp_dense_4h_to_h_bias4) + astype180: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add531, dtype="float16") + add532: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype180, add529) + add533: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add532, add527) + layer_norm181: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add533, gpt_neox_layers_17_input_layernorm_weight4, gpt_neox_layers_17_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims359: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_query_key_value_weight4, axes=None) + matmul359: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm181, permute_dims359, out_dtype="void") + add534: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul359, gpt_neox_layers_17_attention_query_key_value_bias4) + reshape356: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add534, R.shape([batch_size, 1, 48, 128])) + reshape357: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape356, R.shape([batch_size, 48, 128])) + lv90 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape357), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape358: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv90, R.shape([batch_size, 1, 16, 128])) + reshape359: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape358, R.shape([batch_size, 1, 2048])) + permute_dims360: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_dense_weight4, axes=None) + matmul360: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape359, permute_dims360, out_dtype="void") + add535: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul360, gpt_neox_layers_17_attention_dense_bias4) + layer_norm182: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add533, gpt_neox_layers_17_post_attention_layernorm_weight4, gpt_neox_layers_17_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims361: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_h_to_4h_weight4, axes=None) + matmul361: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm182, permute_dims361, out_dtype="float32") + add536: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul361, gpt_neox_layers_17_mlp_dense_h_to_4h_bias4) + gelu89: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add536) + astype181: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu89, dtype="float16") + permute_dims362: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_4h_to_h_weight4, axes=None) + matmul362: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype181, permute_dims362, out_dtype="float32") + add537: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul362, gpt_neox_layers_17_mlp_dense_4h_to_h_bias4) + astype182: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add537, dtype="float16") + add538: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype182, add535) + add539: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add538, add533) + layer_norm183: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add539, gpt_neox_layers_18_input_layernorm_weight4, gpt_neox_layers_18_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims363: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_query_key_value_weight4, axes=None) + matmul363: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm183, permute_dims363, out_dtype="void") + add540: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul363, gpt_neox_layers_18_attention_query_key_value_bias4) + reshape360: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add540, R.shape([batch_size, 1, 48, 128])) + reshape361: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape360, R.shape([batch_size, 48, 128])) + lv91 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape361), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape362: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv91, R.shape([batch_size, 1, 16, 128])) + reshape363: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape362, R.shape([batch_size, 1, 2048])) + permute_dims364: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_dense_weight4, axes=None) + matmul364: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape363, permute_dims364, out_dtype="void") + add541: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul364, gpt_neox_layers_18_attention_dense_bias4) + layer_norm184: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add539, gpt_neox_layers_18_post_attention_layernorm_weight4, gpt_neox_layers_18_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims365: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_h_to_4h_weight4, axes=None) + matmul365: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm184, permute_dims365, out_dtype="float32") + add542: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul365, gpt_neox_layers_18_mlp_dense_h_to_4h_bias4) + gelu90: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add542) + astype183: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu90, dtype="float16") + permute_dims366: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_4h_to_h_weight4, axes=None) + matmul366: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype183, permute_dims366, out_dtype="float32") + add543: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul366, gpt_neox_layers_18_mlp_dense_4h_to_h_bias4) + astype184: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add543, dtype="float16") + add544: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype184, add541) + add545: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add544, add539) + layer_norm185: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add545, gpt_neox_layers_19_input_layernorm_weight4, gpt_neox_layers_19_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims367: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_query_key_value_weight4, axes=None) + matmul367: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm185, permute_dims367, out_dtype="void") + add546: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul367, gpt_neox_layers_19_attention_query_key_value_bias4) + reshape364: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add546, R.shape([batch_size, 1, 48, 128])) + reshape365: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape364, R.shape([batch_size, 48, 128])) + lv92 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape365), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape366: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv92, R.shape([batch_size, 1, 16, 128])) + reshape367: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape366, R.shape([batch_size, 1, 2048])) + permute_dims368: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_dense_weight4, axes=None) + matmul368: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape367, permute_dims368, out_dtype="void") + add547: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul368, gpt_neox_layers_19_attention_dense_bias4) + layer_norm186: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add545, gpt_neox_layers_19_post_attention_layernorm_weight4, gpt_neox_layers_19_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims369: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_h_to_4h_weight4, axes=None) + matmul369: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm186, permute_dims369, out_dtype="float32") + add548: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul369, gpt_neox_layers_19_mlp_dense_h_to_4h_bias4) + gelu91: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add548) + astype185: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu91, dtype="float16") + permute_dims370: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_4h_to_h_weight4, axes=None) + matmul370: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype185, permute_dims370, out_dtype="float32") + add549: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul370, gpt_neox_layers_19_mlp_dense_4h_to_h_bias4) + astype186: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add549, dtype="float16") + add550: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype186, add547) + add551: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add550, add545) + layer_norm187: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add551, gpt_neox_layers_20_input_layernorm_weight4, gpt_neox_layers_20_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims371: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_query_key_value_weight4, axes=None) + matmul371: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm187, permute_dims371, out_dtype="void") + add552: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul371, gpt_neox_layers_20_attention_query_key_value_bias4) + reshape368: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add552, R.shape([batch_size, 1, 48, 128])) + reshape369: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape368, R.shape([batch_size, 48, 128])) + lv93 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape369), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape370: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv93, R.shape([batch_size, 1, 16, 128])) + reshape371: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape370, R.shape([batch_size, 1, 2048])) + permute_dims372: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_dense_weight4, axes=None) + matmul372: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape371, permute_dims372, out_dtype="void") + add553: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul372, gpt_neox_layers_20_attention_dense_bias4) + layer_norm188: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add551, gpt_neox_layers_20_post_attention_layernorm_weight4, gpt_neox_layers_20_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims373: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_h_to_4h_weight4, axes=None) + matmul373: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm188, permute_dims373, out_dtype="float32") + add554: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul373, gpt_neox_layers_20_mlp_dense_h_to_4h_bias4) + gelu92: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add554) + astype187: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu92, dtype="float16") + permute_dims374: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_4h_to_h_weight4, axes=None) + matmul374: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype187, permute_dims374, out_dtype="float32") + add555: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul374, gpt_neox_layers_20_mlp_dense_4h_to_h_bias4) + astype188: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add555, dtype="float16") + add556: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype188, add553) + add557: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add556, add551) + layer_norm189: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add557, gpt_neox_layers_21_input_layernorm_weight4, gpt_neox_layers_21_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims375: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_query_key_value_weight4, axes=None) + matmul375: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm189, permute_dims375, out_dtype="void") + add558: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul375, gpt_neox_layers_21_attention_query_key_value_bias4) + reshape372: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add558, R.shape([batch_size, 1, 48, 128])) + reshape373: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape372, R.shape([batch_size, 48, 128])) + lv94 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape373), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape374: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv94, R.shape([batch_size, 1, 16, 128])) + reshape375: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape374, R.shape([batch_size, 1, 2048])) + permute_dims376: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_dense_weight4, axes=None) + matmul376: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape375, permute_dims376, out_dtype="void") + add559: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul376, gpt_neox_layers_21_attention_dense_bias4) + layer_norm190: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add557, gpt_neox_layers_21_post_attention_layernorm_weight4, gpt_neox_layers_21_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims377: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_h_to_4h_weight4, axes=None) + matmul377: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm190, permute_dims377, out_dtype="float32") + add560: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul377, gpt_neox_layers_21_mlp_dense_h_to_4h_bias4) + gelu93: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add560) + astype189: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu93, dtype="float16") + permute_dims378: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_4h_to_h_weight4, axes=None) + matmul378: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype189, permute_dims378, out_dtype="float32") + add561: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul378, gpt_neox_layers_21_mlp_dense_4h_to_h_bias4) + astype190: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add561, dtype="float16") + add562: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype190, add559) + add563: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add562, add557) + layer_norm191: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add563, gpt_neox_layers_22_input_layernorm_weight4, gpt_neox_layers_22_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims379: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_query_key_value_weight4, axes=None) + matmul379: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm191, permute_dims379, out_dtype="void") + add564: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul379, gpt_neox_layers_22_attention_query_key_value_bias4) + reshape376: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add564, R.shape([batch_size, 1, 48, 128])) + reshape377: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape376, R.shape([batch_size, 48, 128])) + lv95 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape377), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape378: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv95, R.shape([batch_size, 1, 16, 128])) + reshape379: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape378, R.shape([batch_size, 1, 2048])) + permute_dims380: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_dense_weight4, axes=None) + matmul380: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape379, permute_dims380, out_dtype="void") + add565: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul380, gpt_neox_layers_22_attention_dense_bias4) + layer_norm192: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add563, gpt_neox_layers_22_post_attention_layernorm_weight4, gpt_neox_layers_22_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims381: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_h_to_4h_weight4, axes=None) + matmul381: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm192, permute_dims381, out_dtype="float32") + add566: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul381, gpt_neox_layers_22_mlp_dense_h_to_4h_bias4) + gelu94: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add566) + astype191: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu94, dtype="float16") + permute_dims382: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_4h_to_h_weight4, axes=None) + matmul382: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype191, permute_dims382, out_dtype="float32") + add567: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul382, gpt_neox_layers_22_mlp_dense_4h_to_h_bias4) + astype192: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add567, dtype="float16") + add568: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype192, add565) + add569: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add568, add563) + layer_norm193: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add569, gpt_neox_layers_23_input_layernorm_weight4, gpt_neox_layers_23_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims383: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_query_key_value_weight4, axes=None) + matmul383: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.matmul(layer_norm193, permute_dims383, out_dtype="void") + add570: R.Tensor((batch_size, 1, 6144), dtype="float16") = R.add(matmul383, gpt_neox_layers_23_attention_query_key_value_bias4) + reshape380: R.Tensor((batch_size, 1, 48, 128), dtype="float16") = R.reshape(add570, R.shape([batch_size, 1, 48, 128])) + reshape381: R.Tensor((batch_size, 48, 128), dtype="float16") = R.reshape(reshape380, R.shape([batch_size, 48, 128])) + lv96 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape381), out_sinfo=R.Tensor((batch_size, 16, 128), dtype="float16")) + reshape382: R.Tensor((batch_size, 1, 16, 128), dtype="float16") = R.reshape(lv96, R.shape([batch_size, 1, 16, 128])) + reshape383: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape382, R.shape([batch_size, 1, 2048])) + permute_dims384: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_dense_weight4, axes=None) + matmul384: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape383, permute_dims384, out_dtype="void") + add571: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(matmul384, gpt_neox_layers_23_attention_dense_bias4) + layer_norm194: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add569, gpt_neox_layers_23_post_attention_layernorm_weight4, gpt_neox_layers_23_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims385: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_h_to_4h_weight4, axes=None) + matmul385: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.matmul(layer_norm194, permute_dims385, out_dtype="float32") + add572: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.add(matmul385, gpt_neox_layers_23_mlp_dense_h_to_4h_bias4) + gelu95: R.Tensor((batch_size, 1, 8192), dtype="float32") = R.nn.gelu(add572) + astype193: R.Tensor((batch_size, 1, 8192), dtype="float16") = R.astype(gelu95, dtype="float16") + permute_dims386: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_4h_to_h_weight4, axes=None) + matmul386: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(astype193, permute_dims386, out_dtype="float32") + add573: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul386, gpt_neox_layers_23_mlp_dense_4h_to_h_bias4) + astype194: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(add573, dtype="float16") + add574: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(astype194, add571) + add575: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.add(add574, add569) + layer_norm195: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.layer_norm(add575, gpt_neox_final_layer_norm_weight4, gpt_neox_final_layer_norm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims387: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(embed_out_weight4, axes=None) + matmul387: R.Tensor((batch_size, 1, vocab_size), dtype="float16") = R.matmul(layer_norm195, permute_dims387, out_dtype="void") + astype195: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.astype(matmul387, dtype="float32") + gv4: R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype="float32"), R.Object) = astype195, paged_kv_cache + R.output(gv4) + return gv4 + + @R.function + def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor(("vocab_size", 2048), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): + batch_size = T.int64() + vocab_size = T.int64() + seq_len = T.int64() + R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + with R.dataflow(): + gpt_neox_embed_in_weight3: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] + gpt_neox_layers_0_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[1] + gpt_neox_layers_0_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[2] + gpt_neox_layers_0_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[3] + gpt_neox_layers_0_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[4] + gpt_neox_layers_0_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[5] + gpt_neox_layers_0_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[6] + gpt_neox_layers_0_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[7] + gpt_neox_layers_0_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[10] + gpt_neox_layers_0_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[11] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[12] + gpt_neox_layers_1_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[13] + gpt_neox_layers_1_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[14] + gpt_neox_layers_1_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[15] + gpt_neox_layers_1_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[16] + gpt_neox_layers_1_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[17] + gpt_neox_layers_1_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[18] + gpt_neox_layers_1_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[19] + gpt_neox_layers_1_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[20] + gpt_neox_layers_1_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[21] + gpt_neox_layers_1_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[22] + gpt_neox_layers_1_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[23] + gpt_neox_layers_1_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[24] + gpt_neox_layers_2_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[25] + gpt_neox_layers_2_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[26] + gpt_neox_layers_2_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[27] + gpt_neox_layers_2_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[28] + gpt_neox_layers_2_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[29] + gpt_neox_layers_2_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[30] + gpt_neox_layers_2_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[31] + gpt_neox_layers_2_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[32] + gpt_neox_layers_2_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[33] + gpt_neox_layers_2_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[34] + gpt_neox_layers_2_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[35] + gpt_neox_layers_2_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[36] + gpt_neox_layers_3_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[37] + gpt_neox_layers_3_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[38] + gpt_neox_layers_3_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[39] + gpt_neox_layers_3_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[40] + gpt_neox_layers_3_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[41] + gpt_neox_layers_3_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[42] + gpt_neox_layers_3_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[43] + gpt_neox_layers_3_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[44] + gpt_neox_layers_3_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[45] + gpt_neox_layers_3_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[46] + gpt_neox_layers_3_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[47] + gpt_neox_layers_3_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[48] + gpt_neox_layers_4_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[49] + gpt_neox_layers_4_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[50] + gpt_neox_layers_4_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[51] + gpt_neox_layers_4_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[52] + gpt_neox_layers_4_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[53] + gpt_neox_layers_4_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[54] + gpt_neox_layers_4_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[55] + gpt_neox_layers_4_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[56] + gpt_neox_layers_4_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[57] + gpt_neox_layers_4_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[58] + gpt_neox_layers_4_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[59] + gpt_neox_layers_4_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[60] + gpt_neox_layers_5_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[61] + gpt_neox_layers_5_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[62] + gpt_neox_layers_5_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[63] + gpt_neox_layers_5_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[64] + gpt_neox_layers_5_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[65] + gpt_neox_layers_5_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[66] + gpt_neox_layers_5_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[67] + gpt_neox_layers_5_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[68] + gpt_neox_layers_5_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[69] + gpt_neox_layers_5_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[70] + gpt_neox_layers_5_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[71] + gpt_neox_layers_5_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[72] + gpt_neox_layers_6_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[73] + gpt_neox_layers_6_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[74] + gpt_neox_layers_6_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[75] + gpt_neox_layers_6_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[76] + gpt_neox_layers_6_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[77] + gpt_neox_layers_6_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[78] + gpt_neox_layers_6_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[79] + gpt_neox_layers_6_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[80] + gpt_neox_layers_6_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[81] + gpt_neox_layers_6_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[82] + gpt_neox_layers_6_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[83] + gpt_neox_layers_6_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[84] + gpt_neox_layers_7_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[85] + gpt_neox_layers_7_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[86] + gpt_neox_layers_7_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[87] + gpt_neox_layers_7_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[88] + gpt_neox_layers_7_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[89] + gpt_neox_layers_7_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[90] + gpt_neox_layers_7_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[91] + gpt_neox_layers_7_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[92] + gpt_neox_layers_7_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[93] + gpt_neox_layers_7_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[94] + gpt_neox_layers_7_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[95] + gpt_neox_layers_7_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[96] + gpt_neox_layers_8_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[97] + gpt_neox_layers_8_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[98] + gpt_neox_layers_8_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[99] + gpt_neox_layers_8_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[100] + gpt_neox_layers_8_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[101] + gpt_neox_layers_8_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[102] + gpt_neox_layers_8_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[103] + gpt_neox_layers_8_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[104] + gpt_neox_layers_8_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[105] + gpt_neox_layers_8_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[106] + gpt_neox_layers_8_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[107] + gpt_neox_layers_8_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[108] + gpt_neox_layers_9_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[109] + gpt_neox_layers_9_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[110] + gpt_neox_layers_9_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[111] + gpt_neox_layers_9_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[112] + gpt_neox_layers_9_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[113] + gpt_neox_layers_9_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[114] + gpt_neox_layers_9_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[115] + gpt_neox_layers_9_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[116] + gpt_neox_layers_9_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[117] + gpt_neox_layers_9_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[118] + gpt_neox_layers_9_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[119] + gpt_neox_layers_9_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[120] + gpt_neox_layers_10_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[121] + gpt_neox_layers_10_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[122] + gpt_neox_layers_10_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[123] + gpt_neox_layers_10_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[124] + gpt_neox_layers_10_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[125] + gpt_neox_layers_10_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[126] + gpt_neox_layers_10_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[127] + gpt_neox_layers_10_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[128] + gpt_neox_layers_10_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[129] + gpt_neox_layers_10_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[130] + gpt_neox_layers_10_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[131] + gpt_neox_layers_10_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[132] + gpt_neox_layers_11_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[133] + gpt_neox_layers_11_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[134] + gpt_neox_layers_11_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[135] + gpt_neox_layers_11_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[136] + gpt_neox_layers_11_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[137] + gpt_neox_layers_11_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[138] + gpt_neox_layers_11_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[139] + gpt_neox_layers_11_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[140] + gpt_neox_layers_11_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[141] + gpt_neox_layers_11_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[142] + gpt_neox_layers_11_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[143] + gpt_neox_layers_11_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[144] + gpt_neox_layers_12_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[145] + gpt_neox_layers_12_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[146] + gpt_neox_layers_12_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[147] + gpt_neox_layers_12_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[148] + gpt_neox_layers_12_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[149] + gpt_neox_layers_12_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[150] + gpt_neox_layers_12_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[151] + gpt_neox_layers_12_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[152] + gpt_neox_layers_12_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[153] + gpt_neox_layers_12_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[154] + gpt_neox_layers_12_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[155] + gpt_neox_layers_12_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[156] + gpt_neox_layers_13_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[157] + gpt_neox_layers_13_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[158] + gpt_neox_layers_13_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[159] + gpt_neox_layers_13_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[160] + gpt_neox_layers_13_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[161] + gpt_neox_layers_13_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[162] + gpt_neox_layers_13_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[163] + gpt_neox_layers_13_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[164] + gpt_neox_layers_13_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[165] + gpt_neox_layers_13_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[166] + gpt_neox_layers_13_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[167] + gpt_neox_layers_13_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[168] + gpt_neox_layers_14_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[169] + gpt_neox_layers_14_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[170] + gpt_neox_layers_14_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[171] + gpt_neox_layers_14_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[172] + gpt_neox_layers_14_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[173] + gpt_neox_layers_14_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[174] + gpt_neox_layers_14_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[175] + gpt_neox_layers_14_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[176] + gpt_neox_layers_14_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[177] + gpt_neox_layers_14_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[178] + gpt_neox_layers_14_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[179] + gpt_neox_layers_14_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[180] + gpt_neox_layers_15_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[181] + gpt_neox_layers_15_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[182] + gpt_neox_layers_15_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[183] + gpt_neox_layers_15_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[184] + gpt_neox_layers_15_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[185] + gpt_neox_layers_15_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[186] + gpt_neox_layers_15_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[187] + gpt_neox_layers_15_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[188] + gpt_neox_layers_15_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[189] + gpt_neox_layers_15_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[190] + gpt_neox_layers_15_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[191] + gpt_neox_layers_15_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[192] + gpt_neox_layers_16_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[193] + gpt_neox_layers_16_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[194] + gpt_neox_layers_16_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[195] + gpt_neox_layers_16_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[196] + gpt_neox_layers_16_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[197] + gpt_neox_layers_16_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[198] + gpt_neox_layers_16_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[199] + gpt_neox_layers_16_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[200] + gpt_neox_layers_16_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[201] + gpt_neox_layers_16_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[202] + gpt_neox_layers_16_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[203] + gpt_neox_layers_16_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[204] + gpt_neox_layers_17_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[205] + gpt_neox_layers_17_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[206] + gpt_neox_layers_17_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[207] + gpt_neox_layers_17_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[208] + gpt_neox_layers_17_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[209] + gpt_neox_layers_17_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[210] + gpt_neox_layers_17_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[211] + gpt_neox_layers_17_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[212] + gpt_neox_layers_17_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[213] + gpt_neox_layers_17_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[214] + gpt_neox_layers_17_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[215] + gpt_neox_layers_17_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[216] + gpt_neox_layers_18_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[217] + gpt_neox_layers_18_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[218] + gpt_neox_layers_18_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[219] + gpt_neox_layers_18_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[220] + gpt_neox_layers_18_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[221] + gpt_neox_layers_18_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[222] + gpt_neox_layers_18_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[223] + gpt_neox_layers_18_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[224] + gpt_neox_layers_18_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[225] + gpt_neox_layers_18_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[226] + gpt_neox_layers_18_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[227] + gpt_neox_layers_18_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[228] + gpt_neox_layers_19_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[229] + gpt_neox_layers_19_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[230] + gpt_neox_layers_19_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[231] + gpt_neox_layers_19_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[232] + gpt_neox_layers_19_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[233] + gpt_neox_layers_19_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[234] + gpt_neox_layers_19_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[235] + gpt_neox_layers_19_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[236] + gpt_neox_layers_19_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[237] + gpt_neox_layers_19_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[238] + gpt_neox_layers_19_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[239] + gpt_neox_layers_19_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[240] + gpt_neox_layers_20_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[241] + gpt_neox_layers_20_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[242] + gpt_neox_layers_20_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[243] + gpt_neox_layers_20_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[244] + gpt_neox_layers_20_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[245] + gpt_neox_layers_20_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[246] + gpt_neox_layers_20_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[247] + gpt_neox_layers_20_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[248] + gpt_neox_layers_20_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[249] + gpt_neox_layers_20_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[250] + gpt_neox_layers_20_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[251] + gpt_neox_layers_20_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[252] + gpt_neox_layers_21_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[253] + gpt_neox_layers_21_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[254] + gpt_neox_layers_21_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[255] + gpt_neox_layers_21_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[256] + gpt_neox_layers_21_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[257] + gpt_neox_layers_21_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[258] + gpt_neox_layers_21_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[259] + gpt_neox_layers_21_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[260] + gpt_neox_layers_21_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[261] + gpt_neox_layers_21_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[262] + gpt_neox_layers_21_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[263] + gpt_neox_layers_21_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[264] + gpt_neox_layers_22_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[265] + gpt_neox_layers_22_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[266] + gpt_neox_layers_22_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[267] + gpt_neox_layers_22_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[268] + gpt_neox_layers_22_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[269] + gpt_neox_layers_22_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[270] + gpt_neox_layers_22_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[271] + gpt_neox_layers_22_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[272] + gpt_neox_layers_22_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[273] + gpt_neox_layers_22_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[274] + gpt_neox_layers_22_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[275] + gpt_neox_layers_22_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[276] + gpt_neox_layers_23_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[277] + gpt_neox_layers_23_input_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[278] + gpt_neox_layers_23_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[279] + gpt_neox_layers_23_post_attention_layernorm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[280] + gpt_neox_layers_23_attention_query_key_value_weight3: R.Tensor((6144, 2048), dtype="float16") = packed_params[281] + gpt_neox_layers_23_attention_query_key_value_bias3: R.Tensor((6144,), dtype="float16") = packed_params[282] + gpt_neox_layers_23_attention_dense_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[283] + gpt_neox_layers_23_attention_dense_bias3: R.Tensor((2048,), dtype="float16") = packed_params[284] + gpt_neox_layers_23_mlp_dense_h_to_4h_weight3: R.Tensor((8192, 2048), dtype="float16") = packed_params[285] + gpt_neox_layers_23_mlp_dense_h_to_4h_bias3: R.Tensor((8192,), dtype="float32") = packed_params[286] + gpt_neox_layers_23_mlp_dense_4h_to_h_weight3: R.Tensor((2048, 8192), dtype="float16") = packed_params[287] + gpt_neox_layers_23_mlp_dense_4h_to_h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[288] + gpt_neox_final_layer_norm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[289] + gpt_neox_final_layer_norm_bias3: R.Tensor((2048,), dtype="float16") = packed_params[290] + embed_out_weight3: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[291] + layer_norm98: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight3, gpt_neox_layers_0_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims194: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_query_key_value_weight3, axes=None) + matmul194: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm98, permute_dims194, out_dtype="void") + add288: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul194, gpt_neox_layers_0_attention_query_key_value_bias3) + reshape192: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add288, R.shape([1, seq_len, 48, 128])) + reshape193: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape192, R.shape([seq_len, 48, 128])) + lv49 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape193), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape194: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv49, R.shape([1, seq_len, 16, 128])) + reshape195: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape194, R.shape([1, seq_len, 2048])) + permute_dims195: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_dense_weight3, axes=None) + matmul195: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape195, permute_dims195, out_dtype="void") + add289: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul195, gpt_neox_layers_0_attention_dense_bias3) + layer_norm99: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight3, gpt_neox_layers_0_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims196: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, axes=None) + matmul196: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm99, permute_dims196, out_dtype="float32") + add290: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul196, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3) + gelu48: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add290) + astype98: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu48, dtype="float16") + permute_dims197: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, axes=None) + matmul197: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype98, permute_dims197, out_dtype="float32") + add291: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul197, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3) + astype99: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add291, dtype="float16") + add292: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype99, add289) + add293: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add292, input_embeds) + layer_norm100: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add293, gpt_neox_layers_1_input_layernorm_weight3, gpt_neox_layers_1_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims198: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_query_key_value_weight3, axes=None) + matmul198: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm100, permute_dims198, out_dtype="void") + add294: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul198, gpt_neox_layers_1_attention_query_key_value_bias3) + reshape196: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add294, R.shape([1, seq_len, 48, 128])) + reshape197: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape196, R.shape([seq_len, 48, 128])) + lv50 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape197), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape198: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv50, R.shape([1, seq_len, 16, 128])) + reshape199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape198, R.shape([1, seq_len, 2048])) + permute_dims199: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_dense_weight3, axes=None) + matmul199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape199, permute_dims199, out_dtype="void") + add295: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul199, gpt_neox_layers_1_attention_dense_bias3) + layer_norm101: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add293, gpt_neox_layers_1_post_attention_layernorm_weight3, gpt_neox_layers_1_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims200: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_h_to_4h_weight3, axes=None) + matmul200: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm101, permute_dims200, out_dtype="float32") + add296: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul200, gpt_neox_layers_1_mlp_dense_h_to_4h_bias3) + gelu49: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add296) + astype100: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu49, dtype="float16") + permute_dims201: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_4h_to_h_weight3, axes=None) + matmul201: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype100, permute_dims201, out_dtype="float32") + add297: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul201, gpt_neox_layers_1_mlp_dense_4h_to_h_bias3) + astype101: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add297, dtype="float16") + add298: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype101, add295) + add299: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add298, add293) + layer_norm102: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add299, gpt_neox_layers_2_input_layernorm_weight3, gpt_neox_layers_2_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims202: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_query_key_value_weight3, axes=None) + matmul202: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm102, permute_dims202, out_dtype="void") + add300: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul202, gpt_neox_layers_2_attention_query_key_value_bias3) + reshape200: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add300, R.shape([1, seq_len, 48, 128])) + reshape201: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape200, R.shape([seq_len, 48, 128])) + lv51 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape201), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape202: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv51, R.shape([1, seq_len, 16, 128])) + reshape203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape202, R.shape([1, seq_len, 2048])) + permute_dims203: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_dense_weight3, axes=None) + matmul203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape203, permute_dims203, out_dtype="void") + add301: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul203, gpt_neox_layers_2_attention_dense_bias3) + layer_norm103: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add299, gpt_neox_layers_2_post_attention_layernorm_weight3, gpt_neox_layers_2_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims204: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_h_to_4h_weight3, axes=None) + matmul204: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm103, permute_dims204, out_dtype="float32") + add302: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul204, gpt_neox_layers_2_mlp_dense_h_to_4h_bias3) + gelu50: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add302) + astype102: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu50, dtype="float16") + permute_dims205: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_4h_to_h_weight3, axes=None) + matmul205: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype102, permute_dims205, out_dtype="float32") + add303: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul205, gpt_neox_layers_2_mlp_dense_4h_to_h_bias3) + astype103: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add303, dtype="float16") + add304: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype103, add301) + add305: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add304, add299) + layer_norm104: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add305, gpt_neox_layers_3_input_layernorm_weight3, gpt_neox_layers_3_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims206: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_query_key_value_weight3, axes=None) + matmul206: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm104, permute_dims206, out_dtype="void") + add306: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul206, gpt_neox_layers_3_attention_query_key_value_bias3) + reshape204: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add306, R.shape([1, seq_len, 48, 128])) + reshape205: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape204, R.shape([seq_len, 48, 128])) + lv52 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape205), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape206: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv52, R.shape([1, seq_len, 16, 128])) + reshape207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape206, R.shape([1, seq_len, 2048])) + permute_dims207: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_dense_weight3, axes=None) + matmul207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape207, permute_dims207, out_dtype="void") + add307: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul207, gpt_neox_layers_3_attention_dense_bias3) + layer_norm105: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add305, gpt_neox_layers_3_post_attention_layernorm_weight3, gpt_neox_layers_3_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims208: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_h_to_4h_weight3, axes=None) + matmul208: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm105, permute_dims208, out_dtype="float32") + add308: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul208, gpt_neox_layers_3_mlp_dense_h_to_4h_bias3) + gelu51: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add308) + astype104: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu51, dtype="float16") + permute_dims209: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_4h_to_h_weight3, axes=None) + matmul209: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype104, permute_dims209, out_dtype="float32") + add309: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul209, gpt_neox_layers_3_mlp_dense_4h_to_h_bias3) + astype105: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add309, dtype="float16") + add310: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype105, add307) + add311: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add310, add305) + layer_norm106: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add311, gpt_neox_layers_4_input_layernorm_weight3, gpt_neox_layers_4_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims210: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_query_key_value_weight3, axes=None) + matmul210: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm106, permute_dims210, out_dtype="void") + add312: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul210, gpt_neox_layers_4_attention_query_key_value_bias3) + reshape208: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add312, R.shape([1, seq_len, 48, 128])) + reshape209: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape208, R.shape([seq_len, 48, 128])) + lv53 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape209), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape210: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv53, R.shape([1, seq_len, 16, 128])) + reshape211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape210, R.shape([1, seq_len, 2048])) + permute_dims211: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_dense_weight3, axes=None) + matmul211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape211, permute_dims211, out_dtype="void") + add313: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul211, gpt_neox_layers_4_attention_dense_bias3) + layer_norm107: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add311, gpt_neox_layers_4_post_attention_layernorm_weight3, gpt_neox_layers_4_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims212: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_h_to_4h_weight3, axes=None) + matmul212: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm107, permute_dims212, out_dtype="float32") + add314: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul212, gpt_neox_layers_4_mlp_dense_h_to_4h_bias3) + gelu52: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add314) + astype106: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu52, dtype="float16") + permute_dims213: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_4h_to_h_weight3, axes=None) + matmul213: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype106, permute_dims213, out_dtype="float32") + add315: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul213, gpt_neox_layers_4_mlp_dense_4h_to_h_bias3) + astype107: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add315, dtype="float16") + add316: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype107, add313) + add317: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add316, add311) + layer_norm108: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add317, gpt_neox_layers_5_input_layernorm_weight3, gpt_neox_layers_5_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims214: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_query_key_value_weight3, axes=None) + matmul214: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm108, permute_dims214, out_dtype="void") + add318: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul214, gpt_neox_layers_5_attention_query_key_value_bias3) + reshape212: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add318, R.shape([1, seq_len, 48, 128])) + reshape213: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape212, R.shape([seq_len, 48, 128])) + lv54 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape213), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape214: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv54, R.shape([1, seq_len, 16, 128])) + reshape215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape214, R.shape([1, seq_len, 2048])) + permute_dims215: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_dense_weight3, axes=None) + matmul215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape215, permute_dims215, out_dtype="void") + add319: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul215, gpt_neox_layers_5_attention_dense_bias3) + layer_norm109: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add317, gpt_neox_layers_5_post_attention_layernorm_weight3, gpt_neox_layers_5_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims216: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_h_to_4h_weight3, axes=None) + matmul216: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm109, permute_dims216, out_dtype="float32") + add320: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul216, gpt_neox_layers_5_mlp_dense_h_to_4h_bias3) + gelu53: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add320) + astype108: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu53, dtype="float16") + permute_dims217: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_4h_to_h_weight3, axes=None) + matmul217: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype108, permute_dims217, out_dtype="float32") + add321: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul217, gpt_neox_layers_5_mlp_dense_4h_to_h_bias3) + astype109: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add321, dtype="float16") + add322: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype109, add319) + add323: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add322, add317) + layer_norm110: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add323, gpt_neox_layers_6_input_layernorm_weight3, gpt_neox_layers_6_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims218: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_query_key_value_weight3, axes=None) + matmul218: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm110, permute_dims218, out_dtype="void") + add324: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul218, gpt_neox_layers_6_attention_query_key_value_bias3) + reshape216: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add324, R.shape([1, seq_len, 48, 128])) + reshape217: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape216, R.shape([seq_len, 48, 128])) + lv55 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape217), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape218: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv55, R.shape([1, seq_len, 16, 128])) + reshape219: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape218, R.shape([1, seq_len, 2048])) + permute_dims219: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_dense_weight3, axes=None) + matmul219: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape219, permute_dims219, out_dtype="void") + add325: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul219, gpt_neox_layers_6_attention_dense_bias3) + layer_norm111: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add323, gpt_neox_layers_6_post_attention_layernorm_weight3, gpt_neox_layers_6_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims220: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_h_to_4h_weight3, axes=None) + matmul220: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm111, permute_dims220, out_dtype="float32") + add326: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul220, gpt_neox_layers_6_mlp_dense_h_to_4h_bias3) + gelu54: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add326) + astype110: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu54, dtype="float16") + permute_dims221: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_4h_to_h_weight3, axes=None) + matmul221: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype110, permute_dims221, out_dtype="float32") + add327: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul221, gpt_neox_layers_6_mlp_dense_4h_to_h_bias3) + astype111: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add327, dtype="float16") + add328: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype111, add325) + add329: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add328, add323) + layer_norm112: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add329, gpt_neox_layers_7_input_layernorm_weight3, gpt_neox_layers_7_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims222: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_query_key_value_weight3, axes=None) + matmul222: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm112, permute_dims222, out_dtype="void") + add330: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul222, gpt_neox_layers_7_attention_query_key_value_bias3) + reshape220: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add330, R.shape([1, seq_len, 48, 128])) + reshape221: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape220, R.shape([seq_len, 48, 128])) + lv56 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape221), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape222: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv56, R.shape([1, seq_len, 16, 128])) + reshape223: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape222, R.shape([1, seq_len, 2048])) + permute_dims223: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_dense_weight3, axes=None) + matmul223: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape223, permute_dims223, out_dtype="void") + add331: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul223, gpt_neox_layers_7_attention_dense_bias3) + layer_norm113: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add329, gpt_neox_layers_7_post_attention_layernorm_weight3, gpt_neox_layers_7_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims224: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_h_to_4h_weight3, axes=None) + matmul224: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm113, permute_dims224, out_dtype="float32") + add332: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul224, gpt_neox_layers_7_mlp_dense_h_to_4h_bias3) + gelu55: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add332) + astype112: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu55, dtype="float16") + permute_dims225: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_4h_to_h_weight3, axes=None) + matmul225: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype112, permute_dims225, out_dtype="float32") + add333: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul225, gpt_neox_layers_7_mlp_dense_4h_to_h_bias3) + astype113: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add333, dtype="float16") + add334: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype113, add331) + add335: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add334, add329) + layer_norm114: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add335, gpt_neox_layers_8_input_layernorm_weight3, gpt_neox_layers_8_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims226: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_query_key_value_weight3, axes=None) + matmul226: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm114, permute_dims226, out_dtype="void") + add336: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul226, gpt_neox_layers_8_attention_query_key_value_bias3) + reshape224: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add336, R.shape([1, seq_len, 48, 128])) + reshape225: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape224, R.shape([seq_len, 48, 128])) + lv57 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape225), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape226: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv57, R.shape([1, seq_len, 16, 128])) + reshape227: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape226, R.shape([1, seq_len, 2048])) + permute_dims227: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_dense_weight3, axes=None) + matmul227: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape227, permute_dims227, out_dtype="void") + add337: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul227, gpt_neox_layers_8_attention_dense_bias3) + layer_norm115: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add335, gpt_neox_layers_8_post_attention_layernorm_weight3, gpt_neox_layers_8_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims228: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_h_to_4h_weight3, axes=None) + matmul228: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm115, permute_dims228, out_dtype="float32") + add338: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul228, gpt_neox_layers_8_mlp_dense_h_to_4h_bias3) + gelu56: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add338) + astype114: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu56, dtype="float16") + permute_dims229: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_4h_to_h_weight3, axes=None) + matmul229: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype114, permute_dims229, out_dtype="float32") + add339: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul229, gpt_neox_layers_8_mlp_dense_4h_to_h_bias3) + astype115: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add339, dtype="float16") + add340: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype115, add337) + add341: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add340, add335) + layer_norm116: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add341, gpt_neox_layers_9_input_layernorm_weight3, gpt_neox_layers_9_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims230: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_query_key_value_weight3, axes=None) + matmul230: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm116, permute_dims230, out_dtype="void") + add342: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul230, gpt_neox_layers_9_attention_query_key_value_bias3) + reshape228: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add342, R.shape([1, seq_len, 48, 128])) + reshape229: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape228, R.shape([seq_len, 48, 128])) + lv58 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape229), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape230: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv58, R.shape([1, seq_len, 16, 128])) + reshape231: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape230, R.shape([1, seq_len, 2048])) + permute_dims231: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_dense_weight3, axes=None) + matmul231: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape231, permute_dims231, out_dtype="void") + add343: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul231, gpt_neox_layers_9_attention_dense_bias3) + layer_norm117: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add341, gpt_neox_layers_9_post_attention_layernorm_weight3, gpt_neox_layers_9_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims232: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_h_to_4h_weight3, axes=None) + matmul232: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm117, permute_dims232, out_dtype="float32") + add344: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul232, gpt_neox_layers_9_mlp_dense_h_to_4h_bias3) + gelu57: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add344) + astype116: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu57, dtype="float16") + permute_dims233: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_4h_to_h_weight3, axes=None) + matmul233: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype116, permute_dims233, out_dtype="float32") + add345: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul233, gpt_neox_layers_9_mlp_dense_4h_to_h_bias3) + astype117: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add345, dtype="float16") + add346: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype117, add343) + add347: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add346, add341) + layer_norm118: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add347, gpt_neox_layers_10_input_layernorm_weight3, gpt_neox_layers_10_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims234: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_query_key_value_weight3, axes=None) + matmul234: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm118, permute_dims234, out_dtype="void") + add348: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul234, gpt_neox_layers_10_attention_query_key_value_bias3) + reshape232: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add348, R.shape([1, seq_len, 48, 128])) + reshape233: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape232, R.shape([seq_len, 48, 128])) + lv59 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape233), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape234: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv59, R.shape([1, seq_len, 16, 128])) + reshape235: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape234, R.shape([1, seq_len, 2048])) + permute_dims235: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_dense_weight3, axes=None) + matmul235: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape235, permute_dims235, out_dtype="void") + add349: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul235, gpt_neox_layers_10_attention_dense_bias3) + layer_norm119: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add347, gpt_neox_layers_10_post_attention_layernorm_weight3, gpt_neox_layers_10_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims236: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_h_to_4h_weight3, axes=None) + matmul236: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm119, permute_dims236, out_dtype="float32") + add350: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul236, gpt_neox_layers_10_mlp_dense_h_to_4h_bias3) + gelu58: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add350) + astype118: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu58, dtype="float16") + permute_dims237: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_4h_to_h_weight3, axes=None) + matmul237: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype118, permute_dims237, out_dtype="float32") + add351: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul237, gpt_neox_layers_10_mlp_dense_4h_to_h_bias3) + astype119: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add351, dtype="float16") + add352: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype119, add349) + add353: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add352, add347) + layer_norm120: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add353, gpt_neox_layers_11_input_layernorm_weight3, gpt_neox_layers_11_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims238: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_query_key_value_weight3, axes=None) + matmul238: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm120, permute_dims238, out_dtype="void") + add354: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul238, gpt_neox_layers_11_attention_query_key_value_bias3) + reshape236: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add354, R.shape([1, seq_len, 48, 128])) + reshape237: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape236, R.shape([seq_len, 48, 128])) + lv60 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape237), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape238: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv60, R.shape([1, seq_len, 16, 128])) + reshape239: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape238, R.shape([1, seq_len, 2048])) + permute_dims239: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_dense_weight3, axes=None) + matmul239: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape239, permute_dims239, out_dtype="void") + add355: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul239, gpt_neox_layers_11_attention_dense_bias3) + layer_norm121: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add353, gpt_neox_layers_11_post_attention_layernorm_weight3, gpt_neox_layers_11_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims240: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_h_to_4h_weight3, axes=None) + matmul240: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm121, permute_dims240, out_dtype="float32") + add356: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul240, gpt_neox_layers_11_mlp_dense_h_to_4h_bias3) + gelu59: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add356) + astype120: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu59, dtype="float16") + permute_dims241: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_4h_to_h_weight3, axes=None) + matmul241: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype120, permute_dims241, out_dtype="float32") + add357: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul241, gpt_neox_layers_11_mlp_dense_4h_to_h_bias3) + astype121: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add357, dtype="float16") + add358: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype121, add355) + add359: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add358, add353) + layer_norm122: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add359, gpt_neox_layers_12_input_layernorm_weight3, gpt_neox_layers_12_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims242: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_query_key_value_weight3, axes=None) + matmul242: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm122, permute_dims242, out_dtype="void") + add360: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul242, gpt_neox_layers_12_attention_query_key_value_bias3) + reshape240: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add360, R.shape([1, seq_len, 48, 128])) + reshape241: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape240, R.shape([seq_len, 48, 128])) + lv61 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape241), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape242: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv61, R.shape([1, seq_len, 16, 128])) + reshape243: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape242, R.shape([1, seq_len, 2048])) + permute_dims243: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_dense_weight3, axes=None) + matmul243: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape243, permute_dims243, out_dtype="void") + add361: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul243, gpt_neox_layers_12_attention_dense_bias3) + layer_norm123: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add359, gpt_neox_layers_12_post_attention_layernorm_weight3, gpt_neox_layers_12_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims244: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_h_to_4h_weight3, axes=None) + matmul244: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm123, permute_dims244, out_dtype="float32") + add362: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul244, gpt_neox_layers_12_mlp_dense_h_to_4h_bias3) + gelu60: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add362) + astype122: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu60, dtype="float16") + permute_dims245: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_4h_to_h_weight3, axes=None) + matmul245: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype122, permute_dims245, out_dtype="float32") + add363: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul245, gpt_neox_layers_12_mlp_dense_4h_to_h_bias3) + astype123: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add363, dtype="float16") + add364: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype123, add361) + add365: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add364, add359) + layer_norm124: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add365, gpt_neox_layers_13_input_layernorm_weight3, gpt_neox_layers_13_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims246: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_query_key_value_weight3, axes=None) + matmul246: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm124, permute_dims246, out_dtype="void") + add366: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul246, gpt_neox_layers_13_attention_query_key_value_bias3) + reshape244: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add366, R.shape([1, seq_len, 48, 128])) + reshape245: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape244, R.shape([seq_len, 48, 128])) + lv62 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape245), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape246: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv62, R.shape([1, seq_len, 16, 128])) + reshape247: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape246, R.shape([1, seq_len, 2048])) + permute_dims247: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_dense_weight3, axes=None) + matmul247: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape247, permute_dims247, out_dtype="void") + add367: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul247, gpt_neox_layers_13_attention_dense_bias3) + layer_norm125: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add365, gpt_neox_layers_13_post_attention_layernorm_weight3, gpt_neox_layers_13_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims248: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_h_to_4h_weight3, axes=None) + matmul248: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm125, permute_dims248, out_dtype="float32") + add368: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul248, gpt_neox_layers_13_mlp_dense_h_to_4h_bias3) + gelu61: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add368) + astype124: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu61, dtype="float16") + permute_dims249: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_4h_to_h_weight3, axes=None) + matmul249: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype124, permute_dims249, out_dtype="float32") + add369: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul249, gpt_neox_layers_13_mlp_dense_4h_to_h_bias3) + astype125: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add369, dtype="float16") + add370: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype125, add367) + add371: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add370, add365) + layer_norm126: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add371, gpt_neox_layers_14_input_layernorm_weight3, gpt_neox_layers_14_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims250: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_query_key_value_weight3, axes=None) + matmul250: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm126, permute_dims250, out_dtype="void") + add372: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul250, gpt_neox_layers_14_attention_query_key_value_bias3) + reshape248: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add372, R.shape([1, seq_len, 48, 128])) + reshape249: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape248, R.shape([seq_len, 48, 128])) + lv63 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape249), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape250: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv63, R.shape([1, seq_len, 16, 128])) + reshape251: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape250, R.shape([1, seq_len, 2048])) + permute_dims251: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_dense_weight3, axes=None) + matmul251: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape251, permute_dims251, out_dtype="void") + add373: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul251, gpt_neox_layers_14_attention_dense_bias3) + layer_norm127: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add371, gpt_neox_layers_14_post_attention_layernorm_weight3, gpt_neox_layers_14_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims252: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_h_to_4h_weight3, axes=None) + matmul252: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm127, permute_dims252, out_dtype="float32") + add374: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul252, gpt_neox_layers_14_mlp_dense_h_to_4h_bias3) + gelu62: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add374) + astype126: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu62, dtype="float16") + permute_dims253: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_4h_to_h_weight3, axes=None) + matmul253: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype126, permute_dims253, out_dtype="float32") + add375: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul253, gpt_neox_layers_14_mlp_dense_4h_to_h_bias3) + astype127: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add375, dtype="float16") + add376: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype127, add373) + add377: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add376, add371) + layer_norm128: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add377, gpt_neox_layers_15_input_layernorm_weight3, gpt_neox_layers_15_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims254: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_query_key_value_weight3, axes=None) + matmul254: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm128, permute_dims254, out_dtype="void") + add378: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul254, gpt_neox_layers_15_attention_query_key_value_bias3) + reshape252: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add378, R.shape([1, seq_len, 48, 128])) + reshape253: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape252, R.shape([seq_len, 48, 128])) + lv64 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape253), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape254: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv64, R.shape([1, seq_len, 16, 128])) + reshape255: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape254, R.shape([1, seq_len, 2048])) + permute_dims255: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_dense_weight3, axes=None) + matmul255: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape255, permute_dims255, out_dtype="void") + add379: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul255, gpt_neox_layers_15_attention_dense_bias3) + layer_norm129: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add377, gpt_neox_layers_15_post_attention_layernorm_weight3, gpt_neox_layers_15_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims256: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_h_to_4h_weight3, axes=None) + matmul256: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm129, permute_dims256, out_dtype="float32") + add380: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul256, gpt_neox_layers_15_mlp_dense_h_to_4h_bias3) + gelu63: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add380) + astype128: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu63, dtype="float16") + permute_dims257: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_4h_to_h_weight3, axes=None) + matmul257: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype128, permute_dims257, out_dtype="float32") + add381: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul257, gpt_neox_layers_15_mlp_dense_4h_to_h_bias3) + astype129: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add381, dtype="float16") + add382: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype129, add379) + add383: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add382, add377) + layer_norm130: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add383, gpt_neox_layers_16_input_layernorm_weight3, gpt_neox_layers_16_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims258: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_query_key_value_weight3, axes=None) + matmul258: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm130, permute_dims258, out_dtype="void") + add384: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul258, gpt_neox_layers_16_attention_query_key_value_bias3) + reshape256: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add384, R.shape([1, seq_len, 48, 128])) + reshape257: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape256, R.shape([seq_len, 48, 128])) + lv65 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape257), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape258: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv65, R.shape([1, seq_len, 16, 128])) + reshape259: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape258, R.shape([1, seq_len, 2048])) + permute_dims259: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_dense_weight3, axes=None) + matmul259: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape259, permute_dims259, out_dtype="void") + add385: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul259, gpt_neox_layers_16_attention_dense_bias3) + layer_norm131: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add383, gpt_neox_layers_16_post_attention_layernorm_weight3, gpt_neox_layers_16_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims260: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_h_to_4h_weight3, axes=None) + matmul260: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm131, permute_dims260, out_dtype="float32") + add386: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul260, gpt_neox_layers_16_mlp_dense_h_to_4h_bias3) + gelu64: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add386) + astype130: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu64, dtype="float16") + permute_dims261: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_4h_to_h_weight3, axes=None) + matmul261: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype130, permute_dims261, out_dtype="float32") + add387: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul261, gpt_neox_layers_16_mlp_dense_4h_to_h_bias3) + astype131: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add387, dtype="float16") + add388: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype131, add385) + add389: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add388, add383) + layer_norm132: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add389, gpt_neox_layers_17_input_layernorm_weight3, gpt_neox_layers_17_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims262: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_query_key_value_weight3, axes=None) + matmul262: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm132, permute_dims262, out_dtype="void") + add390: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul262, gpt_neox_layers_17_attention_query_key_value_bias3) + reshape260: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add390, R.shape([1, seq_len, 48, 128])) + reshape261: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape260, R.shape([seq_len, 48, 128])) + lv66 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape261), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape262: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv66, R.shape([1, seq_len, 16, 128])) + reshape263: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape262, R.shape([1, seq_len, 2048])) + permute_dims263: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_dense_weight3, axes=None) + matmul263: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape263, permute_dims263, out_dtype="void") + add391: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul263, gpt_neox_layers_17_attention_dense_bias3) + layer_norm133: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add389, gpt_neox_layers_17_post_attention_layernorm_weight3, gpt_neox_layers_17_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims264: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_h_to_4h_weight3, axes=None) + matmul264: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm133, permute_dims264, out_dtype="float32") + add392: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul264, gpt_neox_layers_17_mlp_dense_h_to_4h_bias3) + gelu65: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add392) + astype132: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu65, dtype="float16") + permute_dims265: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_4h_to_h_weight3, axes=None) + matmul265: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype132, permute_dims265, out_dtype="float32") + add393: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul265, gpt_neox_layers_17_mlp_dense_4h_to_h_bias3) + astype133: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add393, dtype="float16") + add394: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype133, add391) + add395: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add394, add389) + layer_norm134: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add395, gpt_neox_layers_18_input_layernorm_weight3, gpt_neox_layers_18_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims266: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_query_key_value_weight3, axes=None) + matmul266: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm134, permute_dims266, out_dtype="void") + add396: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul266, gpt_neox_layers_18_attention_query_key_value_bias3) + reshape264: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add396, R.shape([1, seq_len, 48, 128])) + reshape265: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape264, R.shape([seq_len, 48, 128])) + lv67 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape265), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape266: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv67, R.shape([1, seq_len, 16, 128])) + reshape267: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape266, R.shape([1, seq_len, 2048])) + permute_dims267: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_dense_weight3, axes=None) + matmul267: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape267, permute_dims267, out_dtype="void") + add397: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul267, gpt_neox_layers_18_attention_dense_bias3) + layer_norm135: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add395, gpt_neox_layers_18_post_attention_layernorm_weight3, gpt_neox_layers_18_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims268: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_h_to_4h_weight3, axes=None) + matmul268: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm135, permute_dims268, out_dtype="float32") + add398: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul268, gpt_neox_layers_18_mlp_dense_h_to_4h_bias3) + gelu66: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add398) + astype134: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu66, dtype="float16") + permute_dims269: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_4h_to_h_weight3, axes=None) + matmul269: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype134, permute_dims269, out_dtype="float32") + add399: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul269, gpt_neox_layers_18_mlp_dense_4h_to_h_bias3) + astype135: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add399, dtype="float16") + add400: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype135, add397) + add401: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add400, add395) + layer_norm136: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add401, gpt_neox_layers_19_input_layernorm_weight3, gpt_neox_layers_19_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims270: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_query_key_value_weight3, axes=None) + matmul270: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm136, permute_dims270, out_dtype="void") + add402: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul270, gpt_neox_layers_19_attention_query_key_value_bias3) + reshape268: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add402, R.shape([1, seq_len, 48, 128])) + reshape269: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape268, R.shape([seq_len, 48, 128])) + lv68 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape269), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape270: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv68, R.shape([1, seq_len, 16, 128])) + reshape271: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape270, R.shape([1, seq_len, 2048])) + permute_dims271: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_dense_weight3, axes=None) + matmul271: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape271, permute_dims271, out_dtype="void") + add403: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul271, gpt_neox_layers_19_attention_dense_bias3) + layer_norm137: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add401, gpt_neox_layers_19_post_attention_layernorm_weight3, gpt_neox_layers_19_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims272: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_h_to_4h_weight3, axes=None) + matmul272: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm137, permute_dims272, out_dtype="float32") + add404: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul272, gpt_neox_layers_19_mlp_dense_h_to_4h_bias3) + gelu67: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add404) + astype136: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu67, dtype="float16") + permute_dims273: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_4h_to_h_weight3, axes=None) + matmul273: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype136, permute_dims273, out_dtype="float32") + add405: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul273, gpt_neox_layers_19_mlp_dense_4h_to_h_bias3) + astype137: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add405, dtype="float16") + add406: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype137, add403) + add407: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add406, add401) + layer_norm138: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add407, gpt_neox_layers_20_input_layernorm_weight3, gpt_neox_layers_20_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims274: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_query_key_value_weight3, axes=None) + matmul274: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm138, permute_dims274, out_dtype="void") + add408: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul274, gpt_neox_layers_20_attention_query_key_value_bias3) + reshape272: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add408, R.shape([1, seq_len, 48, 128])) + reshape273: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape272, R.shape([seq_len, 48, 128])) + lv69 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape273), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape274: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv69, R.shape([1, seq_len, 16, 128])) + reshape275: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape274, R.shape([1, seq_len, 2048])) + permute_dims275: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_dense_weight3, axes=None) + matmul275: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape275, permute_dims275, out_dtype="void") + add409: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul275, gpt_neox_layers_20_attention_dense_bias3) + layer_norm139: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add407, gpt_neox_layers_20_post_attention_layernorm_weight3, gpt_neox_layers_20_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims276: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_h_to_4h_weight3, axes=None) + matmul276: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm139, permute_dims276, out_dtype="float32") + add410: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul276, gpt_neox_layers_20_mlp_dense_h_to_4h_bias3) + gelu68: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add410) + astype138: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu68, dtype="float16") + permute_dims277: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_4h_to_h_weight3, axes=None) + matmul277: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype138, permute_dims277, out_dtype="float32") + add411: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul277, gpt_neox_layers_20_mlp_dense_4h_to_h_bias3) + astype139: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add411, dtype="float16") + add412: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype139, add409) + add413: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add412, add407) + layer_norm140: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add413, gpt_neox_layers_21_input_layernorm_weight3, gpt_neox_layers_21_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims278: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_query_key_value_weight3, axes=None) + matmul278: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm140, permute_dims278, out_dtype="void") + add414: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul278, gpt_neox_layers_21_attention_query_key_value_bias3) + reshape276: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add414, R.shape([1, seq_len, 48, 128])) + reshape277: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape276, R.shape([seq_len, 48, 128])) + lv70 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape277), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape278: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv70, R.shape([1, seq_len, 16, 128])) + reshape279: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape278, R.shape([1, seq_len, 2048])) + permute_dims279: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_dense_weight3, axes=None) + matmul279: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape279, permute_dims279, out_dtype="void") + add415: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul279, gpt_neox_layers_21_attention_dense_bias3) + layer_norm141: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add413, gpt_neox_layers_21_post_attention_layernorm_weight3, gpt_neox_layers_21_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims280: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_h_to_4h_weight3, axes=None) + matmul280: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm141, permute_dims280, out_dtype="float32") + add416: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul280, gpt_neox_layers_21_mlp_dense_h_to_4h_bias3) + gelu69: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add416) + astype140: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu69, dtype="float16") + permute_dims281: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_4h_to_h_weight3, axes=None) + matmul281: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype140, permute_dims281, out_dtype="float32") + add417: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul281, gpt_neox_layers_21_mlp_dense_4h_to_h_bias3) + astype141: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add417, dtype="float16") + add418: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype141, add415) + add419: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add418, add413) + layer_norm142: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add419, gpt_neox_layers_22_input_layernorm_weight3, gpt_neox_layers_22_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims282: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_query_key_value_weight3, axes=None) + matmul282: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm142, permute_dims282, out_dtype="void") + add420: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul282, gpt_neox_layers_22_attention_query_key_value_bias3) + reshape280: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add420, R.shape([1, seq_len, 48, 128])) + reshape281: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape280, R.shape([seq_len, 48, 128])) + lv71 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape281), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape282: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv71, R.shape([1, seq_len, 16, 128])) + reshape283: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape282, R.shape([1, seq_len, 2048])) + permute_dims283: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_dense_weight3, axes=None) + matmul283: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape283, permute_dims283, out_dtype="void") + add421: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul283, gpt_neox_layers_22_attention_dense_bias3) + layer_norm143: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add419, gpt_neox_layers_22_post_attention_layernorm_weight3, gpt_neox_layers_22_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims284: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_h_to_4h_weight3, axes=None) + matmul284: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm143, permute_dims284, out_dtype="float32") + add422: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul284, gpt_neox_layers_22_mlp_dense_h_to_4h_bias3) + gelu70: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add422) + astype142: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu70, dtype="float16") + permute_dims285: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_4h_to_h_weight3, axes=None) + matmul285: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype142, permute_dims285, out_dtype="float32") + add423: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul285, gpt_neox_layers_22_mlp_dense_4h_to_h_bias3) + astype143: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add423, dtype="float16") + add424: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype143, add421) + add425: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add424, add419) + layer_norm144: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add425, gpt_neox_layers_23_input_layernorm_weight3, gpt_neox_layers_23_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims286: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_query_key_value_weight3, axes=None) + matmul286: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm144, permute_dims286, out_dtype="void") + add426: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul286, gpt_neox_layers_23_attention_query_key_value_bias3) + reshape284: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add426, R.shape([1, seq_len, 48, 128])) + reshape285: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape284, R.shape([seq_len, 48, 128])) + lv72 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape285), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape286: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv72, R.shape([1, seq_len, 16, 128])) + reshape287: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape286, R.shape([1, seq_len, 2048])) + permute_dims287: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_dense_weight3, axes=None) + matmul287: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape287, permute_dims287, out_dtype="void") + add427: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul287, gpt_neox_layers_23_attention_dense_bias3) + layer_norm145: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add425, gpt_neox_layers_23_post_attention_layernorm_weight3, gpt_neox_layers_23_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims288: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_h_to_4h_weight3, axes=None) + matmul288: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm145, permute_dims288, out_dtype="float32") + add428: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul288, gpt_neox_layers_23_mlp_dense_h_to_4h_bias3) + gelu71: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add428) + astype144: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu71, dtype="float16") + permute_dims289: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_4h_to_h_weight3, axes=None) + matmul289: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype144, permute_dims289, out_dtype="float32") + add429: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul289, gpt_neox_layers_23_mlp_dense_4h_to_h_bias3) + astype145: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add429, dtype="float16") + add430: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype145, add427) + add431: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add430, add425) + layer_norm146: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add431, gpt_neox_final_layer_norm_weight3, gpt_neox_final_layer_norm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + take1: R.Tensor((1, batch_size, 2048), dtype="float16") = R.take(layer_norm146, logit_positions, axis=1) + permute_dims290: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(embed_out_weight3, axes=None) + matmul290: R.Tensor((1, batch_size, vocab_size), dtype="float16") = R.matmul(take1, permute_dims290, out_dtype="void") + astype146: R.Tensor((1, batch_size, vocab_size), dtype="float32") = R.astype(matmul290, dtype="float32") + gv3: R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype="float32"), R.Object) = astype146, paged_kv_cache + R.output(gv3) + return gv3 + + @R.function + def batch_verify(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor(("vocab_size", 2048), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): + seq_len = T.int64() + vocab_size = T.int64() + R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + with R.dataflow(): + gpt_neox_embed_in_weight5: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] + gpt_neox_layers_0_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[1] + gpt_neox_layers_0_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[2] + gpt_neox_layers_0_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[3] + gpt_neox_layers_0_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[4] + gpt_neox_layers_0_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[5] + gpt_neox_layers_0_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[6] + gpt_neox_layers_0_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[7] + gpt_neox_layers_0_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[10] + gpt_neox_layers_0_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[11] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[12] + gpt_neox_layers_1_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[13] + gpt_neox_layers_1_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[14] + gpt_neox_layers_1_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[15] + gpt_neox_layers_1_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[16] + gpt_neox_layers_1_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[17] + gpt_neox_layers_1_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[18] + gpt_neox_layers_1_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[19] + gpt_neox_layers_1_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[20] + gpt_neox_layers_1_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[21] + gpt_neox_layers_1_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[22] + gpt_neox_layers_1_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[23] + gpt_neox_layers_1_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[24] + gpt_neox_layers_2_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[25] + gpt_neox_layers_2_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[26] + gpt_neox_layers_2_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[27] + gpt_neox_layers_2_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[28] + gpt_neox_layers_2_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[29] + gpt_neox_layers_2_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[30] + gpt_neox_layers_2_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[31] + gpt_neox_layers_2_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[32] + gpt_neox_layers_2_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[33] + gpt_neox_layers_2_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[34] + gpt_neox_layers_2_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[35] + gpt_neox_layers_2_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[36] + gpt_neox_layers_3_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[37] + gpt_neox_layers_3_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[38] + gpt_neox_layers_3_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[39] + gpt_neox_layers_3_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[40] + gpt_neox_layers_3_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[41] + gpt_neox_layers_3_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[42] + gpt_neox_layers_3_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[43] + gpt_neox_layers_3_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[44] + gpt_neox_layers_3_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[45] + gpt_neox_layers_3_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[46] + gpt_neox_layers_3_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[47] + gpt_neox_layers_3_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[48] + gpt_neox_layers_4_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[49] + gpt_neox_layers_4_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[50] + gpt_neox_layers_4_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[51] + gpt_neox_layers_4_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[52] + gpt_neox_layers_4_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[53] + gpt_neox_layers_4_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[54] + gpt_neox_layers_4_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[55] + gpt_neox_layers_4_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[56] + gpt_neox_layers_4_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[57] + gpt_neox_layers_4_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[58] + gpt_neox_layers_4_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[59] + gpt_neox_layers_4_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[60] + gpt_neox_layers_5_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[61] + gpt_neox_layers_5_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[62] + gpt_neox_layers_5_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[63] + gpt_neox_layers_5_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[64] + gpt_neox_layers_5_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[65] + gpt_neox_layers_5_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[66] + gpt_neox_layers_5_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[67] + gpt_neox_layers_5_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[68] + gpt_neox_layers_5_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[69] + gpt_neox_layers_5_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[70] + gpt_neox_layers_5_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[71] + gpt_neox_layers_5_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[72] + gpt_neox_layers_6_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[73] + gpt_neox_layers_6_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[74] + gpt_neox_layers_6_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[75] + gpt_neox_layers_6_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[76] + gpt_neox_layers_6_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[77] + gpt_neox_layers_6_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[78] + gpt_neox_layers_6_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[79] + gpt_neox_layers_6_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[80] + gpt_neox_layers_6_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[81] + gpt_neox_layers_6_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[82] + gpt_neox_layers_6_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[83] + gpt_neox_layers_6_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[84] + gpt_neox_layers_7_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[85] + gpt_neox_layers_7_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[86] + gpt_neox_layers_7_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[87] + gpt_neox_layers_7_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[88] + gpt_neox_layers_7_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[89] + gpt_neox_layers_7_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[90] + gpt_neox_layers_7_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[91] + gpt_neox_layers_7_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[92] + gpt_neox_layers_7_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[93] + gpt_neox_layers_7_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[94] + gpt_neox_layers_7_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[95] + gpt_neox_layers_7_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[96] + gpt_neox_layers_8_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[97] + gpt_neox_layers_8_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[98] + gpt_neox_layers_8_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[99] + gpt_neox_layers_8_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[100] + gpt_neox_layers_8_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[101] + gpt_neox_layers_8_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[102] + gpt_neox_layers_8_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[103] + gpt_neox_layers_8_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[104] + gpt_neox_layers_8_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[105] + gpt_neox_layers_8_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[106] + gpt_neox_layers_8_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[107] + gpt_neox_layers_8_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[108] + gpt_neox_layers_9_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[109] + gpt_neox_layers_9_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[110] + gpt_neox_layers_9_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[111] + gpt_neox_layers_9_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[112] + gpt_neox_layers_9_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[113] + gpt_neox_layers_9_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[114] + gpt_neox_layers_9_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[115] + gpt_neox_layers_9_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[116] + gpt_neox_layers_9_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[117] + gpt_neox_layers_9_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[118] + gpt_neox_layers_9_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[119] + gpt_neox_layers_9_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[120] + gpt_neox_layers_10_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[121] + gpt_neox_layers_10_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[122] + gpt_neox_layers_10_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[123] + gpt_neox_layers_10_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[124] + gpt_neox_layers_10_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[125] + gpt_neox_layers_10_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[126] + gpt_neox_layers_10_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[127] + gpt_neox_layers_10_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[128] + gpt_neox_layers_10_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[129] + gpt_neox_layers_10_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[130] + gpt_neox_layers_10_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[131] + gpt_neox_layers_10_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[132] + gpt_neox_layers_11_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[133] + gpt_neox_layers_11_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[134] + gpt_neox_layers_11_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[135] + gpt_neox_layers_11_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[136] + gpt_neox_layers_11_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[137] + gpt_neox_layers_11_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[138] + gpt_neox_layers_11_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[139] + gpt_neox_layers_11_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[140] + gpt_neox_layers_11_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[141] + gpt_neox_layers_11_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[142] + gpt_neox_layers_11_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[143] + gpt_neox_layers_11_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[144] + gpt_neox_layers_12_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[145] + gpt_neox_layers_12_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[146] + gpt_neox_layers_12_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[147] + gpt_neox_layers_12_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[148] + gpt_neox_layers_12_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[149] + gpt_neox_layers_12_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[150] + gpt_neox_layers_12_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[151] + gpt_neox_layers_12_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[152] + gpt_neox_layers_12_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[153] + gpt_neox_layers_12_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[154] + gpt_neox_layers_12_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[155] + gpt_neox_layers_12_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[156] + gpt_neox_layers_13_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[157] + gpt_neox_layers_13_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[158] + gpt_neox_layers_13_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[159] + gpt_neox_layers_13_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[160] + gpt_neox_layers_13_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[161] + gpt_neox_layers_13_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[162] + gpt_neox_layers_13_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[163] + gpt_neox_layers_13_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[164] + gpt_neox_layers_13_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[165] + gpt_neox_layers_13_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[166] + gpt_neox_layers_13_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[167] + gpt_neox_layers_13_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[168] + gpt_neox_layers_14_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[169] + gpt_neox_layers_14_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[170] + gpt_neox_layers_14_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[171] + gpt_neox_layers_14_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[172] + gpt_neox_layers_14_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[173] + gpt_neox_layers_14_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[174] + gpt_neox_layers_14_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[175] + gpt_neox_layers_14_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[176] + gpt_neox_layers_14_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[177] + gpt_neox_layers_14_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[178] + gpt_neox_layers_14_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[179] + gpt_neox_layers_14_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[180] + gpt_neox_layers_15_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[181] + gpt_neox_layers_15_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[182] + gpt_neox_layers_15_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[183] + gpt_neox_layers_15_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[184] + gpt_neox_layers_15_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[185] + gpt_neox_layers_15_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[186] + gpt_neox_layers_15_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[187] + gpt_neox_layers_15_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[188] + gpt_neox_layers_15_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[189] + gpt_neox_layers_15_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[190] + gpt_neox_layers_15_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[191] + gpt_neox_layers_15_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[192] + gpt_neox_layers_16_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[193] + gpt_neox_layers_16_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[194] + gpt_neox_layers_16_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[195] + gpt_neox_layers_16_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[196] + gpt_neox_layers_16_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[197] + gpt_neox_layers_16_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[198] + gpt_neox_layers_16_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[199] + gpt_neox_layers_16_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[200] + gpt_neox_layers_16_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[201] + gpt_neox_layers_16_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[202] + gpt_neox_layers_16_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[203] + gpt_neox_layers_16_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[204] + gpt_neox_layers_17_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[205] + gpt_neox_layers_17_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[206] + gpt_neox_layers_17_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[207] + gpt_neox_layers_17_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[208] + gpt_neox_layers_17_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[209] + gpt_neox_layers_17_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[210] + gpt_neox_layers_17_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[211] + gpt_neox_layers_17_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[212] + gpt_neox_layers_17_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[213] + gpt_neox_layers_17_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[214] + gpt_neox_layers_17_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[215] + gpt_neox_layers_17_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[216] + gpt_neox_layers_18_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[217] + gpt_neox_layers_18_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[218] + gpt_neox_layers_18_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[219] + gpt_neox_layers_18_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[220] + gpt_neox_layers_18_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[221] + gpt_neox_layers_18_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[222] + gpt_neox_layers_18_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[223] + gpt_neox_layers_18_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[224] + gpt_neox_layers_18_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[225] + gpt_neox_layers_18_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[226] + gpt_neox_layers_18_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[227] + gpt_neox_layers_18_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[228] + gpt_neox_layers_19_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[229] + gpt_neox_layers_19_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[230] + gpt_neox_layers_19_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[231] + gpt_neox_layers_19_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[232] + gpt_neox_layers_19_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[233] + gpt_neox_layers_19_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[234] + gpt_neox_layers_19_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[235] + gpt_neox_layers_19_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[236] + gpt_neox_layers_19_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[237] + gpt_neox_layers_19_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[238] + gpt_neox_layers_19_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[239] + gpt_neox_layers_19_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[240] + gpt_neox_layers_20_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[241] + gpt_neox_layers_20_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[242] + gpt_neox_layers_20_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[243] + gpt_neox_layers_20_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[244] + gpt_neox_layers_20_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[245] + gpt_neox_layers_20_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[246] + gpt_neox_layers_20_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[247] + gpt_neox_layers_20_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[248] + gpt_neox_layers_20_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[249] + gpt_neox_layers_20_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[250] + gpt_neox_layers_20_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[251] + gpt_neox_layers_20_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[252] + gpt_neox_layers_21_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[253] + gpt_neox_layers_21_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[254] + gpt_neox_layers_21_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[255] + gpt_neox_layers_21_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[256] + gpt_neox_layers_21_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[257] + gpt_neox_layers_21_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[258] + gpt_neox_layers_21_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[259] + gpt_neox_layers_21_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[260] + gpt_neox_layers_21_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[261] + gpt_neox_layers_21_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[262] + gpt_neox_layers_21_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[263] + gpt_neox_layers_21_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[264] + gpt_neox_layers_22_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[265] + gpt_neox_layers_22_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[266] + gpt_neox_layers_22_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[267] + gpt_neox_layers_22_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[268] + gpt_neox_layers_22_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[269] + gpt_neox_layers_22_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[270] + gpt_neox_layers_22_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[271] + gpt_neox_layers_22_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[272] + gpt_neox_layers_22_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[273] + gpt_neox_layers_22_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[274] + gpt_neox_layers_22_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[275] + gpt_neox_layers_22_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[276] + gpt_neox_layers_23_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[277] + gpt_neox_layers_23_input_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[278] + gpt_neox_layers_23_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[279] + gpt_neox_layers_23_post_attention_layernorm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[280] + gpt_neox_layers_23_attention_query_key_value_weight5: R.Tensor((6144, 2048), dtype="float16") = packed_params[281] + gpt_neox_layers_23_attention_query_key_value_bias5: R.Tensor((6144,), dtype="float16") = packed_params[282] + gpt_neox_layers_23_attention_dense_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[283] + gpt_neox_layers_23_attention_dense_bias5: R.Tensor((2048,), dtype="float16") = packed_params[284] + gpt_neox_layers_23_mlp_dense_h_to_4h_weight5: R.Tensor((8192, 2048), dtype="float16") = packed_params[285] + gpt_neox_layers_23_mlp_dense_h_to_4h_bias5: R.Tensor((8192,), dtype="float32") = packed_params[286] + gpt_neox_layers_23_mlp_dense_4h_to_h_weight5: R.Tensor((2048, 8192), dtype="float16") = packed_params[287] + gpt_neox_layers_23_mlp_dense_4h_to_h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[288] + gpt_neox_final_layer_norm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[289] + gpt_neox_final_layer_norm_bias5: R.Tensor((2048,), dtype="float16") = packed_params[290] + embed_out_weight5: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[291] + layer_norm196: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight5, gpt_neox_layers_0_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims388: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_query_key_value_weight5, axes=None) + matmul388: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm196, permute_dims388, out_dtype="void") + add576: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul388, gpt_neox_layers_0_attention_query_key_value_bias5) + reshape384: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add576, R.shape([1, seq_len, 48, 128])) + reshape385: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape384, R.shape([seq_len, 48, 128])) + lv97 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape385), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape386: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv97, R.shape([1, seq_len, 16, 128])) + reshape387: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape386, R.shape([1, seq_len, 2048])) + permute_dims389: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_dense_weight5, axes=None) + matmul389: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape387, permute_dims389, out_dtype="void") + add577: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul389, gpt_neox_layers_0_attention_dense_bias5) + layer_norm197: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight5, gpt_neox_layers_0_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims390: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_h_to_4h_weight5, axes=None) + matmul390: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm197, permute_dims390, out_dtype="float32") + add578: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul390, gpt_neox_layers_0_mlp_dense_h_to_4h_bias5) + gelu96: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add578) + astype196: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu96, dtype="float16") + permute_dims391: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_4h_to_h_weight5, axes=None) + matmul391: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype196, permute_dims391, out_dtype="float32") + add579: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul391, gpt_neox_layers_0_mlp_dense_4h_to_h_bias5) + astype197: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add579, dtype="float16") + add580: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype197, add577) + add581: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add580, input_embeds) + layer_norm198: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add581, gpt_neox_layers_1_input_layernorm_weight5, gpt_neox_layers_1_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims392: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_query_key_value_weight5, axes=None) + matmul392: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm198, permute_dims392, out_dtype="void") + add582: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul392, gpt_neox_layers_1_attention_query_key_value_bias5) + reshape388: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add582, R.shape([1, seq_len, 48, 128])) + reshape389: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape388, R.shape([seq_len, 48, 128])) + lv98 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape389), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape390: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv98, R.shape([1, seq_len, 16, 128])) + reshape391: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape390, R.shape([1, seq_len, 2048])) + permute_dims393: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_dense_weight5, axes=None) + matmul393: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape391, permute_dims393, out_dtype="void") + add583: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul393, gpt_neox_layers_1_attention_dense_bias5) + layer_norm199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add581, gpt_neox_layers_1_post_attention_layernorm_weight5, gpt_neox_layers_1_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims394: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_h_to_4h_weight5, axes=None) + matmul394: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm199, permute_dims394, out_dtype="float32") + add584: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul394, gpt_neox_layers_1_mlp_dense_h_to_4h_bias5) + gelu97: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add584) + astype198: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu97, dtype="float16") + permute_dims395: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_4h_to_h_weight5, axes=None) + matmul395: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype198, permute_dims395, out_dtype="float32") + add585: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul395, gpt_neox_layers_1_mlp_dense_4h_to_h_bias5) + astype199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add585, dtype="float16") + add586: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype199, add583) + add587: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add586, add581) + layer_norm200: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add587, gpt_neox_layers_2_input_layernorm_weight5, gpt_neox_layers_2_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims396: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_query_key_value_weight5, axes=None) + matmul396: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm200, permute_dims396, out_dtype="void") + add588: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul396, gpt_neox_layers_2_attention_query_key_value_bias5) + reshape392: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add588, R.shape([1, seq_len, 48, 128])) + reshape393: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape392, R.shape([seq_len, 48, 128])) + lv99 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape393), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape394: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv99, R.shape([1, seq_len, 16, 128])) + reshape395: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape394, R.shape([1, seq_len, 2048])) + permute_dims397: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_dense_weight5, axes=None) + matmul397: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape395, permute_dims397, out_dtype="void") + add589: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul397, gpt_neox_layers_2_attention_dense_bias5) + layer_norm201: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add587, gpt_neox_layers_2_post_attention_layernorm_weight5, gpt_neox_layers_2_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims398: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_h_to_4h_weight5, axes=None) + matmul398: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm201, permute_dims398, out_dtype="float32") + add590: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul398, gpt_neox_layers_2_mlp_dense_h_to_4h_bias5) + gelu98: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add590) + astype200: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu98, dtype="float16") + permute_dims399: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_4h_to_h_weight5, axes=None) + matmul399: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype200, permute_dims399, out_dtype="float32") + add591: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul399, gpt_neox_layers_2_mlp_dense_4h_to_h_bias5) + astype201: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add591, dtype="float16") + add592: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype201, add589) + add593: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add592, add587) + layer_norm202: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add593, gpt_neox_layers_3_input_layernorm_weight5, gpt_neox_layers_3_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims400: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_query_key_value_weight5, axes=None) + matmul400: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm202, permute_dims400, out_dtype="void") + add594: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul400, gpt_neox_layers_3_attention_query_key_value_bias5) + reshape396: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add594, R.shape([1, seq_len, 48, 128])) + reshape397: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape396, R.shape([seq_len, 48, 128])) + lv100 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape397), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape398: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv100, R.shape([1, seq_len, 16, 128])) + reshape399: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape398, R.shape([1, seq_len, 2048])) + permute_dims401: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_dense_weight5, axes=None) + matmul401: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape399, permute_dims401, out_dtype="void") + add595: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul401, gpt_neox_layers_3_attention_dense_bias5) + layer_norm203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add593, gpt_neox_layers_3_post_attention_layernorm_weight5, gpt_neox_layers_3_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims402: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_h_to_4h_weight5, axes=None) + matmul402: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm203, permute_dims402, out_dtype="float32") + add596: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul402, gpt_neox_layers_3_mlp_dense_h_to_4h_bias5) + gelu99: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add596) + astype202: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu99, dtype="float16") + permute_dims403: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_4h_to_h_weight5, axes=None) + matmul403: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype202, permute_dims403, out_dtype="float32") + add597: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul403, gpt_neox_layers_3_mlp_dense_4h_to_h_bias5) + astype203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add597, dtype="float16") + add598: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype203, add595) + add599: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add598, add593) + layer_norm204: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add599, gpt_neox_layers_4_input_layernorm_weight5, gpt_neox_layers_4_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims404: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_query_key_value_weight5, axes=None) + matmul404: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm204, permute_dims404, out_dtype="void") + add600: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul404, gpt_neox_layers_4_attention_query_key_value_bias5) + reshape400: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add600, R.shape([1, seq_len, 48, 128])) + reshape401: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape400, R.shape([seq_len, 48, 128])) + lv101 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape401), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape402: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv101, R.shape([1, seq_len, 16, 128])) + reshape403: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape402, R.shape([1, seq_len, 2048])) + permute_dims405: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_dense_weight5, axes=None) + matmul405: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape403, permute_dims405, out_dtype="void") + add601: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul405, gpt_neox_layers_4_attention_dense_bias5) + layer_norm205: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add599, gpt_neox_layers_4_post_attention_layernorm_weight5, gpt_neox_layers_4_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims406: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_h_to_4h_weight5, axes=None) + matmul406: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm205, permute_dims406, out_dtype="float32") + add602: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul406, gpt_neox_layers_4_mlp_dense_h_to_4h_bias5) + gelu100: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add602) + astype204: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu100, dtype="float16") + permute_dims407: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_4h_to_h_weight5, axes=None) + matmul407: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype204, permute_dims407, out_dtype="float32") + add603: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul407, gpt_neox_layers_4_mlp_dense_4h_to_h_bias5) + astype205: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add603, dtype="float16") + add604: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype205, add601) + add605: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add604, add599) + layer_norm206: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add605, gpt_neox_layers_5_input_layernorm_weight5, gpt_neox_layers_5_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims408: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_query_key_value_weight5, axes=None) + matmul408: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm206, permute_dims408, out_dtype="void") + add606: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul408, gpt_neox_layers_5_attention_query_key_value_bias5) + reshape404: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add606, R.shape([1, seq_len, 48, 128])) + reshape405: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape404, R.shape([seq_len, 48, 128])) + lv102 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape405), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape406: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv102, R.shape([1, seq_len, 16, 128])) + reshape407: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape406, R.shape([1, seq_len, 2048])) + permute_dims409: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_dense_weight5, axes=None) + matmul409: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape407, permute_dims409, out_dtype="void") + add607: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul409, gpt_neox_layers_5_attention_dense_bias5) + layer_norm207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add605, gpt_neox_layers_5_post_attention_layernorm_weight5, gpt_neox_layers_5_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims410: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_h_to_4h_weight5, axes=None) + matmul410: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm207, permute_dims410, out_dtype="float32") + add608: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul410, gpt_neox_layers_5_mlp_dense_h_to_4h_bias5) + gelu101: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add608) + astype206: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu101, dtype="float16") + permute_dims411: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_4h_to_h_weight5, axes=None) + matmul411: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype206, permute_dims411, out_dtype="float32") + add609: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul411, gpt_neox_layers_5_mlp_dense_4h_to_h_bias5) + astype207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add609, dtype="float16") + add610: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype207, add607) + add611: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add610, add605) + layer_norm208: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add611, gpt_neox_layers_6_input_layernorm_weight5, gpt_neox_layers_6_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims412: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_query_key_value_weight5, axes=None) + matmul412: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm208, permute_dims412, out_dtype="void") + add612: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul412, gpt_neox_layers_6_attention_query_key_value_bias5) + reshape408: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add612, R.shape([1, seq_len, 48, 128])) + reshape409: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape408, R.shape([seq_len, 48, 128])) + lv103 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape409), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape410: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv103, R.shape([1, seq_len, 16, 128])) + reshape411: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape410, R.shape([1, seq_len, 2048])) + permute_dims413: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_dense_weight5, axes=None) + matmul413: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape411, permute_dims413, out_dtype="void") + add613: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul413, gpt_neox_layers_6_attention_dense_bias5) + layer_norm209: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add611, gpt_neox_layers_6_post_attention_layernorm_weight5, gpt_neox_layers_6_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims414: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_h_to_4h_weight5, axes=None) + matmul414: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm209, permute_dims414, out_dtype="float32") + add614: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul414, gpt_neox_layers_6_mlp_dense_h_to_4h_bias5) + gelu102: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add614) + astype208: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu102, dtype="float16") + permute_dims415: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_4h_to_h_weight5, axes=None) + matmul415: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype208, permute_dims415, out_dtype="float32") + add615: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul415, gpt_neox_layers_6_mlp_dense_4h_to_h_bias5) + astype209: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add615, dtype="float16") + add616: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype209, add613) + add617: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add616, add611) + layer_norm210: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add617, gpt_neox_layers_7_input_layernorm_weight5, gpt_neox_layers_7_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims416: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_query_key_value_weight5, axes=None) + matmul416: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm210, permute_dims416, out_dtype="void") + add618: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul416, gpt_neox_layers_7_attention_query_key_value_bias5) + reshape412: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add618, R.shape([1, seq_len, 48, 128])) + reshape413: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape412, R.shape([seq_len, 48, 128])) + lv104 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape413), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape414: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv104, R.shape([1, seq_len, 16, 128])) + reshape415: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape414, R.shape([1, seq_len, 2048])) + permute_dims417: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_dense_weight5, axes=None) + matmul417: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape415, permute_dims417, out_dtype="void") + add619: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul417, gpt_neox_layers_7_attention_dense_bias5) + layer_norm211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add617, gpt_neox_layers_7_post_attention_layernorm_weight5, gpt_neox_layers_7_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims418: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_h_to_4h_weight5, axes=None) + matmul418: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm211, permute_dims418, out_dtype="float32") + add620: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul418, gpt_neox_layers_7_mlp_dense_h_to_4h_bias5) + gelu103: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add620) + astype210: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu103, dtype="float16") + permute_dims419: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_4h_to_h_weight5, axes=None) + matmul419: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype210, permute_dims419, out_dtype="float32") + add621: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul419, gpt_neox_layers_7_mlp_dense_4h_to_h_bias5) + astype211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add621, dtype="float16") + add622: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype211, add619) + add623: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add622, add617) + layer_norm212: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add623, gpt_neox_layers_8_input_layernorm_weight5, gpt_neox_layers_8_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims420: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_query_key_value_weight5, axes=None) + matmul420: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm212, permute_dims420, out_dtype="void") + add624: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul420, gpt_neox_layers_8_attention_query_key_value_bias5) + reshape416: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add624, R.shape([1, seq_len, 48, 128])) + reshape417: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape416, R.shape([seq_len, 48, 128])) + lv105 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape417), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape418: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv105, R.shape([1, seq_len, 16, 128])) + reshape419: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape418, R.shape([1, seq_len, 2048])) + permute_dims421: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_dense_weight5, axes=None) + matmul421: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape419, permute_dims421, out_dtype="void") + add625: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul421, gpt_neox_layers_8_attention_dense_bias5) + layer_norm213: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add623, gpt_neox_layers_8_post_attention_layernorm_weight5, gpt_neox_layers_8_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims422: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_h_to_4h_weight5, axes=None) + matmul422: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm213, permute_dims422, out_dtype="float32") + add626: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul422, gpt_neox_layers_8_mlp_dense_h_to_4h_bias5) + gelu104: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add626) + astype212: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu104, dtype="float16") + permute_dims423: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_4h_to_h_weight5, axes=None) + matmul423: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype212, permute_dims423, out_dtype="float32") + add627: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul423, gpt_neox_layers_8_mlp_dense_4h_to_h_bias5) + astype213: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add627, dtype="float16") + add628: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype213, add625) + add629: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add628, add623) + layer_norm214: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add629, gpt_neox_layers_9_input_layernorm_weight5, gpt_neox_layers_9_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims424: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_query_key_value_weight5, axes=None) + matmul424: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm214, permute_dims424, out_dtype="void") + add630: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul424, gpt_neox_layers_9_attention_query_key_value_bias5) + reshape420: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add630, R.shape([1, seq_len, 48, 128])) + reshape421: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape420, R.shape([seq_len, 48, 128])) + lv106 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape421), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape422: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv106, R.shape([1, seq_len, 16, 128])) + reshape423: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape422, R.shape([1, seq_len, 2048])) + permute_dims425: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_dense_weight5, axes=None) + matmul425: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape423, permute_dims425, out_dtype="void") + add631: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul425, gpt_neox_layers_9_attention_dense_bias5) + layer_norm215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add629, gpt_neox_layers_9_post_attention_layernorm_weight5, gpt_neox_layers_9_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims426: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_h_to_4h_weight5, axes=None) + matmul426: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm215, permute_dims426, out_dtype="float32") + add632: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul426, gpt_neox_layers_9_mlp_dense_h_to_4h_bias5) + gelu105: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add632) + astype214: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu105, dtype="float16") + permute_dims427: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_4h_to_h_weight5, axes=None) + matmul427: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype214, permute_dims427, out_dtype="float32") + add633: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul427, gpt_neox_layers_9_mlp_dense_4h_to_h_bias5) + astype215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add633, dtype="float16") + add634: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype215, add631) + add635: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add634, add629) + layer_norm216: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add635, gpt_neox_layers_10_input_layernorm_weight5, gpt_neox_layers_10_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims428: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_query_key_value_weight5, axes=None) + matmul428: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm216, permute_dims428, out_dtype="void") + add636: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul428, gpt_neox_layers_10_attention_query_key_value_bias5) + reshape424: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add636, R.shape([1, seq_len, 48, 128])) + reshape425: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape424, R.shape([seq_len, 48, 128])) + lv107 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape425), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape426: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv107, R.shape([1, seq_len, 16, 128])) + reshape427: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape426, R.shape([1, seq_len, 2048])) + permute_dims429: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_dense_weight5, axes=None) + matmul429: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape427, permute_dims429, out_dtype="void") + add637: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul429, gpt_neox_layers_10_attention_dense_bias5) + layer_norm217: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add635, gpt_neox_layers_10_post_attention_layernorm_weight5, gpt_neox_layers_10_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims430: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_h_to_4h_weight5, axes=None) + matmul430: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm217, permute_dims430, out_dtype="float32") + add638: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul430, gpt_neox_layers_10_mlp_dense_h_to_4h_bias5) + gelu106: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add638) + astype216: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu106, dtype="float16") + permute_dims431: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_4h_to_h_weight5, axes=None) + matmul431: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype216, permute_dims431, out_dtype="float32") + add639: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul431, gpt_neox_layers_10_mlp_dense_4h_to_h_bias5) + astype217: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add639, dtype="float16") + add640: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype217, add637) + add641: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add640, add635) + layer_norm218: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add641, gpt_neox_layers_11_input_layernorm_weight5, gpt_neox_layers_11_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims432: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_query_key_value_weight5, axes=None) + matmul432: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm218, permute_dims432, out_dtype="void") + add642: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul432, gpt_neox_layers_11_attention_query_key_value_bias5) + reshape428: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add642, R.shape([1, seq_len, 48, 128])) + reshape429: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape428, R.shape([seq_len, 48, 128])) + lv108 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape429), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape430: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv108, R.shape([1, seq_len, 16, 128])) + reshape431: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape430, R.shape([1, seq_len, 2048])) + permute_dims433: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_dense_weight5, axes=None) + matmul433: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape431, permute_dims433, out_dtype="void") + add643: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul433, gpt_neox_layers_11_attention_dense_bias5) + layer_norm219: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add641, gpt_neox_layers_11_post_attention_layernorm_weight5, gpt_neox_layers_11_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims434: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_h_to_4h_weight5, axes=None) + matmul434: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm219, permute_dims434, out_dtype="float32") + add644: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul434, gpt_neox_layers_11_mlp_dense_h_to_4h_bias5) + gelu107: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add644) + astype218: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu107, dtype="float16") + permute_dims435: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_4h_to_h_weight5, axes=None) + matmul435: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype218, permute_dims435, out_dtype="float32") + add645: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul435, gpt_neox_layers_11_mlp_dense_4h_to_h_bias5) + astype219: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add645, dtype="float16") + add646: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype219, add643) + add647: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add646, add641) + layer_norm220: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add647, gpt_neox_layers_12_input_layernorm_weight5, gpt_neox_layers_12_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims436: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_query_key_value_weight5, axes=None) + matmul436: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm220, permute_dims436, out_dtype="void") + add648: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul436, gpt_neox_layers_12_attention_query_key_value_bias5) + reshape432: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add648, R.shape([1, seq_len, 48, 128])) + reshape433: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape432, R.shape([seq_len, 48, 128])) + lv109 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape433), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape434: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv109, R.shape([1, seq_len, 16, 128])) + reshape435: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape434, R.shape([1, seq_len, 2048])) + permute_dims437: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_dense_weight5, axes=None) + matmul437: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape435, permute_dims437, out_dtype="void") + add649: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul437, gpt_neox_layers_12_attention_dense_bias5) + layer_norm221: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add647, gpt_neox_layers_12_post_attention_layernorm_weight5, gpt_neox_layers_12_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims438: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_h_to_4h_weight5, axes=None) + matmul438: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm221, permute_dims438, out_dtype="float32") + add650: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul438, gpt_neox_layers_12_mlp_dense_h_to_4h_bias5) + gelu108: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add650) + astype220: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu108, dtype="float16") + permute_dims439: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_4h_to_h_weight5, axes=None) + matmul439: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype220, permute_dims439, out_dtype="float32") + add651: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul439, gpt_neox_layers_12_mlp_dense_4h_to_h_bias5) + astype221: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add651, dtype="float16") + add652: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype221, add649) + add653: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add652, add647) + layer_norm222: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add653, gpt_neox_layers_13_input_layernorm_weight5, gpt_neox_layers_13_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims440: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_query_key_value_weight5, axes=None) + matmul440: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm222, permute_dims440, out_dtype="void") + add654: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul440, gpt_neox_layers_13_attention_query_key_value_bias5) + reshape436: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add654, R.shape([1, seq_len, 48, 128])) + reshape437: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape436, R.shape([seq_len, 48, 128])) + lv110 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape437), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape438: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv110, R.shape([1, seq_len, 16, 128])) + reshape439: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape438, R.shape([1, seq_len, 2048])) + permute_dims441: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_dense_weight5, axes=None) + matmul441: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape439, permute_dims441, out_dtype="void") + add655: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul441, gpt_neox_layers_13_attention_dense_bias5) + layer_norm223: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add653, gpt_neox_layers_13_post_attention_layernorm_weight5, gpt_neox_layers_13_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims442: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_h_to_4h_weight5, axes=None) + matmul442: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm223, permute_dims442, out_dtype="float32") + add656: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul442, gpt_neox_layers_13_mlp_dense_h_to_4h_bias5) + gelu109: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add656) + astype222: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu109, dtype="float16") + permute_dims443: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_4h_to_h_weight5, axes=None) + matmul443: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype222, permute_dims443, out_dtype="float32") + add657: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul443, gpt_neox_layers_13_mlp_dense_4h_to_h_bias5) + astype223: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add657, dtype="float16") + add658: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype223, add655) + add659: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add658, add653) + layer_norm224: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add659, gpt_neox_layers_14_input_layernorm_weight5, gpt_neox_layers_14_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims444: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_query_key_value_weight5, axes=None) + matmul444: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm224, permute_dims444, out_dtype="void") + add660: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul444, gpt_neox_layers_14_attention_query_key_value_bias5) + reshape440: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add660, R.shape([1, seq_len, 48, 128])) + reshape441: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape440, R.shape([seq_len, 48, 128])) + lv111 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape441), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape442: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv111, R.shape([1, seq_len, 16, 128])) + reshape443: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape442, R.shape([1, seq_len, 2048])) + permute_dims445: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_dense_weight5, axes=None) + matmul445: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape443, permute_dims445, out_dtype="void") + add661: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul445, gpt_neox_layers_14_attention_dense_bias5) + layer_norm225: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add659, gpt_neox_layers_14_post_attention_layernorm_weight5, gpt_neox_layers_14_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims446: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_h_to_4h_weight5, axes=None) + matmul446: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm225, permute_dims446, out_dtype="float32") + add662: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul446, gpt_neox_layers_14_mlp_dense_h_to_4h_bias5) + gelu110: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add662) + astype224: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu110, dtype="float16") + permute_dims447: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_4h_to_h_weight5, axes=None) + matmul447: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype224, permute_dims447, out_dtype="float32") + add663: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul447, gpt_neox_layers_14_mlp_dense_4h_to_h_bias5) + astype225: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add663, dtype="float16") + add664: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype225, add661) + add665: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add664, add659) + layer_norm226: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add665, gpt_neox_layers_15_input_layernorm_weight5, gpt_neox_layers_15_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims448: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_query_key_value_weight5, axes=None) + matmul448: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm226, permute_dims448, out_dtype="void") + add666: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul448, gpt_neox_layers_15_attention_query_key_value_bias5) + reshape444: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add666, R.shape([1, seq_len, 48, 128])) + reshape445: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape444, R.shape([seq_len, 48, 128])) + lv112 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape445), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape446: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv112, R.shape([1, seq_len, 16, 128])) + reshape447: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape446, R.shape([1, seq_len, 2048])) + permute_dims449: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_dense_weight5, axes=None) + matmul449: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape447, permute_dims449, out_dtype="void") + add667: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul449, gpt_neox_layers_15_attention_dense_bias5) + layer_norm227: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add665, gpt_neox_layers_15_post_attention_layernorm_weight5, gpt_neox_layers_15_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims450: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_h_to_4h_weight5, axes=None) + matmul450: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm227, permute_dims450, out_dtype="float32") + add668: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul450, gpt_neox_layers_15_mlp_dense_h_to_4h_bias5) + gelu111: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add668) + astype226: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu111, dtype="float16") + permute_dims451: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_4h_to_h_weight5, axes=None) + matmul451: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype226, permute_dims451, out_dtype="float32") + add669: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul451, gpt_neox_layers_15_mlp_dense_4h_to_h_bias5) + astype227: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add669, dtype="float16") + add670: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype227, add667) + add671: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add670, add665) + layer_norm228: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add671, gpt_neox_layers_16_input_layernorm_weight5, gpt_neox_layers_16_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims452: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_query_key_value_weight5, axes=None) + matmul452: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm228, permute_dims452, out_dtype="void") + add672: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul452, gpt_neox_layers_16_attention_query_key_value_bias5) + reshape448: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add672, R.shape([1, seq_len, 48, 128])) + reshape449: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape448, R.shape([seq_len, 48, 128])) + lv113 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape449), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape450: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv113, R.shape([1, seq_len, 16, 128])) + reshape451: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape450, R.shape([1, seq_len, 2048])) + permute_dims453: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_dense_weight5, axes=None) + matmul453: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape451, permute_dims453, out_dtype="void") + add673: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul453, gpt_neox_layers_16_attention_dense_bias5) + layer_norm229: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add671, gpt_neox_layers_16_post_attention_layernorm_weight5, gpt_neox_layers_16_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims454: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_h_to_4h_weight5, axes=None) + matmul454: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm229, permute_dims454, out_dtype="float32") + add674: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul454, gpt_neox_layers_16_mlp_dense_h_to_4h_bias5) + gelu112: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add674) + astype228: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu112, dtype="float16") + permute_dims455: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_4h_to_h_weight5, axes=None) + matmul455: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype228, permute_dims455, out_dtype="float32") + add675: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul455, gpt_neox_layers_16_mlp_dense_4h_to_h_bias5) + astype229: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add675, dtype="float16") + add676: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype229, add673) + add677: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add676, add671) + layer_norm230: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add677, gpt_neox_layers_17_input_layernorm_weight5, gpt_neox_layers_17_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims456: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_query_key_value_weight5, axes=None) + matmul456: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm230, permute_dims456, out_dtype="void") + add678: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul456, gpt_neox_layers_17_attention_query_key_value_bias5) + reshape452: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add678, R.shape([1, seq_len, 48, 128])) + reshape453: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape452, R.shape([seq_len, 48, 128])) + lv114 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape453), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape454: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv114, R.shape([1, seq_len, 16, 128])) + reshape455: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape454, R.shape([1, seq_len, 2048])) + permute_dims457: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_dense_weight5, axes=None) + matmul457: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape455, permute_dims457, out_dtype="void") + add679: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul457, gpt_neox_layers_17_attention_dense_bias5) + layer_norm231: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add677, gpt_neox_layers_17_post_attention_layernorm_weight5, gpt_neox_layers_17_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims458: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_h_to_4h_weight5, axes=None) + matmul458: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm231, permute_dims458, out_dtype="float32") + add680: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul458, gpt_neox_layers_17_mlp_dense_h_to_4h_bias5) + gelu113: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add680) + astype230: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu113, dtype="float16") + permute_dims459: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_4h_to_h_weight5, axes=None) + matmul459: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype230, permute_dims459, out_dtype="float32") + add681: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul459, gpt_neox_layers_17_mlp_dense_4h_to_h_bias5) + astype231: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add681, dtype="float16") + add682: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype231, add679) + add683: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add682, add677) + layer_norm232: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add683, gpt_neox_layers_18_input_layernorm_weight5, gpt_neox_layers_18_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims460: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_query_key_value_weight5, axes=None) + matmul460: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm232, permute_dims460, out_dtype="void") + add684: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul460, gpt_neox_layers_18_attention_query_key_value_bias5) + reshape456: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add684, R.shape([1, seq_len, 48, 128])) + reshape457: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape456, R.shape([seq_len, 48, 128])) + lv115 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape457), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape458: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv115, R.shape([1, seq_len, 16, 128])) + reshape459: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape458, R.shape([1, seq_len, 2048])) + permute_dims461: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_dense_weight5, axes=None) + matmul461: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape459, permute_dims461, out_dtype="void") + add685: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul461, gpt_neox_layers_18_attention_dense_bias5) + layer_norm233: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add683, gpt_neox_layers_18_post_attention_layernorm_weight5, gpt_neox_layers_18_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims462: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_h_to_4h_weight5, axes=None) + matmul462: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm233, permute_dims462, out_dtype="float32") + add686: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul462, gpt_neox_layers_18_mlp_dense_h_to_4h_bias5) + gelu114: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add686) + astype232: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu114, dtype="float16") + permute_dims463: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_4h_to_h_weight5, axes=None) + matmul463: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype232, permute_dims463, out_dtype="float32") + add687: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul463, gpt_neox_layers_18_mlp_dense_4h_to_h_bias5) + astype233: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add687, dtype="float16") + add688: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype233, add685) + add689: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add688, add683) + layer_norm234: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add689, gpt_neox_layers_19_input_layernorm_weight5, gpt_neox_layers_19_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims464: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_query_key_value_weight5, axes=None) + matmul464: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm234, permute_dims464, out_dtype="void") + add690: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul464, gpt_neox_layers_19_attention_query_key_value_bias5) + reshape460: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add690, R.shape([1, seq_len, 48, 128])) + reshape461: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape460, R.shape([seq_len, 48, 128])) + lv116 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape461), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape462: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv116, R.shape([1, seq_len, 16, 128])) + reshape463: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape462, R.shape([1, seq_len, 2048])) + permute_dims465: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_dense_weight5, axes=None) + matmul465: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape463, permute_dims465, out_dtype="void") + add691: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul465, gpt_neox_layers_19_attention_dense_bias5) + layer_norm235: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add689, gpt_neox_layers_19_post_attention_layernorm_weight5, gpt_neox_layers_19_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims466: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_h_to_4h_weight5, axes=None) + matmul466: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm235, permute_dims466, out_dtype="float32") + add692: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul466, gpt_neox_layers_19_mlp_dense_h_to_4h_bias5) + gelu115: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add692) + astype234: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu115, dtype="float16") + permute_dims467: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_4h_to_h_weight5, axes=None) + matmul467: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype234, permute_dims467, out_dtype="float32") + add693: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul467, gpt_neox_layers_19_mlp_dense_4h_to_h_bias5) + astype235: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add693, dtype="float16") + add694: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype235, add691) + add695: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add694, add689) + layer_norm236: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add695, gpt_neox_layers_20_input_layernorm_weight5, gpt_neox_layers_20_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims468: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_query_key_value_weight5, axes=None) + matmul468: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm236, permute_dims468, out_dtype="void") + add696: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul468, gpt_neox_layers_20_attention_query_key_value_bias5) + reshape464: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add696, R.shape([1, seq_len, 48, 128])) + reshape465: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape464, R.shape([seq_len, 48, 128])) + lv117 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape465), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape466: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv117, R.shape([1, seq_len, 16, 128])) + reshape467: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape466, R.shape([1, seq_len, 2048])) + permute_dims469: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_dense_weight5, axes=None) + matmul469: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape467, permute_dims469, out_dtype="void") + add697: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul469, gpt_neox_layers_20_attention_dense_bias5) + layer_norm237: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add695, gpt_neox_layers_20_post_attention_layernorm_weight5, gpt_neox_layers_20_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims470: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_h_to_4h_weight5, axes=None) + matmul470: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm237, permute_dims470, out_dtype="float32") + add698: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul470, gpt_neox_layers_20_mlp_dense_h_to_4h_bias5) + gelu116: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add698) + astype236: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu116, dtype="float16") + permute_dims471: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_4h_to_h_weight5, axes=None) + matmul471: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype236, permute_dims471, out_dtype="float32") + add699: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul471, gpt_neox_layers_20_mlp_dense_4h_to_h_bias5) + astype237: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add699, dtype="float16") + add700: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype237, add697) + add701: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add700, add695) + layer_norm238: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add701, gpt_neox_layers_21_input_layernorm_weight5, gpt_neox_layers_21_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims472: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_query_key_value_weight5, axes=None) + matmul472: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm238, permute_dims472, out_dtype="void") + add702: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul472, gpt_neox_layers_21_attention_query_key_value_bias5) + reshape468: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add702, R.shape([1, seq_len, 48, 128])) + reshape469: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape468, R.shape([seq_len, 48, 128])) + lv118 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape469), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape470: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv118, R.shape([1, seq_len, 16, 128])) + reshape471: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape470, R.shape([1, seq_len, 2048])) + permute_dims473: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_dense_weight5, axes=None) + matmul473: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape471, permute_dims473, out_dtype="void") + add703: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul473, gpt_neox_layers_21_attention_dense_bias5) + layer_norm239: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add701, gpt_neox_layers_21_post_attention_layernorm_weight5, gpt_neox_layers_21_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims474: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_h_to_4h_weight5, axes=None) + matmul474: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm239, permute_dims474, out_dtype="float32") + add704: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul474, gpt_neox_layers_21_mlp_dense_h_to_4h_bias5) + gelu117: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add704) + astype238: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu117, dtype="float16") + permute_dims475: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_4h_to_h_weight5, axes=None) + matmul475: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype238, permute_dims475, out_dtype="float32") + add705: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul475, gpt_neox_layers_21_mlp_dense_4h_to_h_bias5) + astype239: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add705, dtype="float16") + add706: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype239, add703) + add707: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add706, add701) + layer_norm240: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add707, gpt_neox_layers_22_input_layernorm_weight5, gpt_neox_layers_22_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims476: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_query_key_value_weight5, axes=None) + matmul476: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm240, permute_dims476, out_dtype="void") + add708: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul476, gpt_neox_layers_22_attention_query_key_value_bias5) + reshape472: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add708, R.shape([1, seq_len, 48, 128])) + reshape473: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape472, R.shape([seq_len, 48, 128])) + lv119 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape473), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape474: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv119, R.shape([1, seq_len, 16, 128])) + reshape475: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape474, R.shape([1, seq_len, 2048])) + permute_dims477: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_dense_weight5, axes=None) + matmul477: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape475, permute_dims477, out_dtype="void") + add709: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul477, gpt_neox_layers_22_attention_dense_bias5) + layer_norm241: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add707, gpt_neox_layers_22_post_attention_layernorm_weight5, gpt_neox_layers_22_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims478: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_h_to_4h_weight5, axes=None) + matmul478: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm241, permute_dims478, out_dtype="float32") + add710: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul478, gpt_neox_layers_22_mlp_dense_h_to_4h_bias5) + gelu118: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add710) + astype240: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu118, dtype="float16") + permute_dims479: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_4h_to_h_weight5, axes=None) + matmul479: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype240, permute_dims479, out_dtype="float32") + add711: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul479, gpt_neox_layers_22_mlp_dense_4h_to_h_bias5) + astype241: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add711, dtype="float16") + add712: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype241, add709) + add713: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add712, add707) + layer_norm242: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add713, gpt_neox_layers_23_input_layernorm_weight5, gpt_neox_layers_23_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims480: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_query_key_value_weight5, axes=None) + matmul480: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm242, permute_dims480, out_dtype="void") + add714: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul480, gpt_neox_layers_23_attention_query_key_value_bias5) + reshape476: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add714, R.shape([1, seq_len, 48, 128])) + reshape477: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape476, R.shape([seq_len, 48, 128])) + lv120 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape477), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape478: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv120, R.shape([1, seq_len, 16, 128])) + reshape479: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape478, R.shape([1, seq_len, 2048])) + permute_dims481: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_dense_weight5, axes=None) + matmul481: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape479, permute_dims481, out_dtype="void") + add715: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul481, gpt_neox_layers_23_attention_dense_bias5) + layer_norm243: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add713, gpt_neox_layers_23_post_attention_layernorm_weight5, gpt_neox_layers_23_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims482: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_h_to_4h_weight5, axes=None) + matmul482: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm243, permute_dims482, out_dtype="float32") + add716: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul482, gpt_neox_layers_23_mlp_dense_h_to_4h_bias5) + gelu119: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add716) + astype242: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu119, dtype="float16") + permute_dims483: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_4h_to_h_weight5, axes=None) + matmul483: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype242, permute_dims483, out_dtype="float32") + add717: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul483, gpt_neox_layers_23_mlp_dense_4h_to_h_bias5) + astype243: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add717, dtype="float16") + add718: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype243, add715) + add719: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add718, add713) + layer_norm244: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add719, gpt_neox_final_layer_norm_weight5, gpt_neox_final_layer_norm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims484: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(embed_out_weight5, axes=None) + matmul484: R.Tensor((1, seq_len, vocab_size), dtype="float16") = R.matmul(layer_norm244, permute_dims484, out_dtype="void") + astype244: R.Tensor((1, seq_len, vocab_size), dtype="float32") = R.astype(matmul484, dtype="float32") + gv5: R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype="float32"), R.Object) = astype244, paged_kv_cache + R.output(gv5) + return gv5 + + @R.function + def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: + max_batch_size = T.int64() + max_total_seq_len = T.int64() + prefill_chunk_size = T.int64() + page_size = T.int64() + support_sliding_window = T.int64() + R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + cls = Module + gv: R.Tensor((), dtype="float16") = R.zeros(R.shape([]), dtype="float16") + paged_kv_cache: R.Object = R.call_pure_packed("vm.builtin.paged_attention_kv_cache_create_reduced", R.shape([max_batch_size, max_total_seq_len, prefill_chunk_size, page_size, support_sliding_window]), R.shape([0, 24]), R.prim_value(16), R.prim_value(16), R.prim_value(128), R.prim_value(1), R.prim_value(1), R.prim_value(10000), gv, cls.tir_kv_cache_transpose_append, cls.batch_prefill_paged_kv, cls.batch_decode_paged_kv, cls.batch_prefill_paged_kv_sliding_window, cls.batch_decode_paged_kv_sliding_window, cls.batch_prefill_ragged_kv, cls.merge_state_inplace, cls.fused_rope, cls.copy_single_page, cls.tir_kv_cache_debug_get_kv, cls.compact_kv_copy, cls.batch_tree_attn, cls.tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) + return paged_kv_cache + + @R.function + def decode(input_embed: R.Tensor((1, 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor(("vocab_size", 2048), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): + vocab_size = T.int64() + R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + with R.dataflow(): + gpt_neox_embed_in_weight2: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] + gpt_neox_layers_0_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[1] + gpt_neox_layers_0_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[2] + gpt_neox_layers_0_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[3] + gpt_neox_layers_0_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[4] + gpt_neox_layers_0_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[5] + gpt_neox_layers_0_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[6] + gpt_neox_layers_0_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[7] + gpt_neox_layers_0_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[10] + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[11] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[12] + gpt_neox_layers_1_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[13] + gpt_neox_layers_1_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[14] + gpt_neox_layers_1_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[15] + gpt_neox_layers_1_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[16] + gpt_neox_layers_1_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[17] + gpt_neox_layers_1_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[18] + gpt_neox_layers_1_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[19] + gpt_neox_layers_1_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[20] + gpt_neox_layers_1_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[21] + gpt_neox_layers_1_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[22] + gpt_neox_layers_1_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[23] + gpt_neox_layers_1_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[24] + gpt_neox_layers_2_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[25] + gpt_neox_layers_2_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[26] + gpt_neox_layers_2_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[27] + gpt_neox_layers_2_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[28] + gpt_neox_layers_2_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[29] + gpt_neox_layers_2_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[30] + gpt_neox_layers_2_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[31] + gpt_neox_layers_2_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[32] + gpt_neox_layers_2_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[33] + gpt_neox_layers_2_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[34] + gpt_neox_layers_2_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[35] + gpt_neox_layers_2_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[36] + gpt_neox_layers_3_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[37] + gpt_neox_layers_3_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[38] + gpt_neox_layers_3_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[39] + gpt_neox_layers_3_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[40] + gpt_neox_layers_3_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[41] + gpt_neox_layers_3_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[42] + gpt_neox_layers_3_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[43] + gpt_neox_layers_3_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[44] + gpt_neox_layers_3_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[45] + gpt_neox_layers_3_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[46] + gpt_neox_layers_3_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[47] + gpt_neox_layers_3_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[48] + gpt_neox_layers_4_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[49] + gpt_neox_layers_4_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[50] + gpt_neox_layers_4_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[51] + gpt_neox_layers_4_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[52] + gpt_neox_layers_4_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[53] + gpt_neox_layers_4_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[54] + gpt_neox_layers_4_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[55] + gpt_neox_layers_4_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[56] + gpt_neox_layers_4_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[57] + gpt_neox_layers_4_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[58] + gpt_neox_layers_4_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[59] + gpt_neox_layers_4_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[60] + gpt_neox_layers_5_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[61] + gpt_neox_layers_5_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[62] + gpt_neox_layers_5_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[63] + gpt_neox_layers_5_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[64] + gpt_neox_layers_5_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[65] + gpt_neox_layers_5_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[66] + gpt_neox_layers_5_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[67] + gpt_neox_layers_5_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[68] + gpt_neox_layers_5_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[69] + gpt_neox_layers_5_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[70] + gpt_neox_layers_5_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[71] + gpt_neox_layers_5_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[72] + gpt_neox_layers_6_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[73] + gpt_neox_layers_6_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[74] + gpt_neox_layers_6_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[75] + gpt_neox_layers_6_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[76] + gpt_neox_layers_6_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[77] + gpt_neox_layers_6_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[78] + gpt_neox_layers_6_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[79] + gpt_neox_layers_6_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[80] + gpt_neox_layers_6_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[81] + gpt_neox_layers_6_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[82] + gpt_neox_layers_6_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[83] + gpt_neox_layers_6_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[84] + gpt_neox_layers_7_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[85] + gpt_neox_layers_7_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[86] + gpt_neox_layers_7_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[87] + gpt_neox_layers_7_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[88] + gpt_neox_layers_7_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[89] + gpt_neox_layers_7_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[90] + gpt_neox_layers_7_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[91] + gpt_neox_layers_7_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[92] + gpt_neox_layers_7_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[93] + gpt_neox_layers_7_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[94] + gpt_neox_layers_7_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[95] + gpt_neox_layers_7_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[96] + gpt_neox_layers_8_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[97] + gpt_neox_layers_8_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[98] + gpt_neox_layers_8_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[99] + gpt_neox_layers_8_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[100] + gpt_neox_layers_8_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[101] + gpt_neox_layers_8_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[102] + gpt_neox_layers_8_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[103] + gpt_neox_layers_8_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[104] + gpt_neox_layers_8_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[105] + gpt_neox_layers_8_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[106] + gpt_neox_layers_8_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[107] + gpt_neox_layers_8_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[108] + gpt_neox_layers_9_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[109] + gpt_neox_layers_9_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[110] + gpt_neox_layers_9_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[111] + gpt_neox_layers_9_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[112] + gpt_neox_layers_9_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[113] + gpt_neox_layers_9_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[114] + gpt_neox_layers_9_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[115] + gpt_neox_layers_9_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[116] + gpt_neox_layers_9_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[117] + gpt_neox_layers_9_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[118] + gpt_neox_layers_9_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[119] + gpt_neox_layers_9_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[120] + gpt_neox_layers_10_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[121] + gpt_neox_layers_10_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[122] + gpt_neox_layers_10_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[123] + gpt_neox_layers_10_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[124] + gpt_neox_layers_10_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[125] + gpt_neox_layers_10_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[126] + gpt_neox_layers_10_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[127] + gpt_neox_layers_10_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[128] + gpt_neox_layers_10_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[129] + gpt_neox_layers_10_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[130] + gpt_neox_layers_10_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[131] + gpt_neox_layers_10_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[132] + gpt_neox_layers_11_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[133] + gpt_neox_layers_11_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[134] + gpt_neox_layers_11_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[135] + gpt_neox_layers_11_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[136] + gpt_neox_layers_11_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[137] + gpt_neox_layers_11_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[138] + gpt_neox_layers_11_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[139] + gpt_neox_layers_11_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[140] + gpt_neox_layers_11_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[141] + gpt_neox_layers_11_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[142] + gpt_neox_layers_11_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[143] + gpt_neox_layers_11_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[144] + gpt_neox_layers_12_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[145] + gpt_neox_layers_12_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[146] + gpt_neox_layers_12_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[147] + gpt_neox_layers_12_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[148] + gpt_neox_layers_12_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[149] + gpt_neox_layers_12_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[150] + gpt_neox_layers_12_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[151] + gpt_neox_layers_12_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[152] + gpt_neox_layers_12_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[153] + gpt_neox_layers_12_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[154] + gpt_neox_layers_12_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[155] + gpt_neox_layers_12_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[156] + gpt_neox_layers_13_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[157] + gpt_neox_layers_13_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[158] + gpt_neox_layers_13_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[159] + gpt_neox_layers_13_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[160] + gpt_neox_layers_13_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[161] + gpt_neox_layers_13_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[162] + gpt_neox_layers_13_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[163] + gpt_neox_layers_13_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[164] + gpt_neox_layers_13_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[165] + gpt_neox_layers_13_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[166] + gpt_neox_layers_13_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[167] + gpt_neox_layers_13_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[168] + gpt_neox_layers_14_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[169] + gpt_neox_layers_14_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[170] + gpt_neox_layers_14_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[171] + gpt_neox_layers_14_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[172] + gpt_neox_layers_14_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[173] + gpt_neox_layers_14_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[174] + gpt_neox_layers_14_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[175] + gpt_neox_layers_14_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[176] + gpt_neox_layers_14_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[177] + gpt_neox_layers_14_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[178] + gpt_neox_layers_14_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[179] + gpt_neox_layers_14_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[180] + gpt_neox_layers_15_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[181] + gpt_neox_layers_15_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[182] + gpt_neox_layers_15_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[183] + gpt_neox_layers_15_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[184] + gpt_neox_layers_15_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[185] + gpt_neox_layers_15_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[186] + gpt_neox_layers_15_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[187] + gpt_neox_layers_15_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[188] + gpt_neox_layers_15_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[189] + gpt_neox_layers_15_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[190] + gpt_neox_layers_15_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[191] + gpt_neox_layers_15_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[192] + gpt_neox_layers_16_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[193] + gpt_neox_layers_16_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[194] + gpt_neox_layers_16_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[195] + gpt_neox_layers_16_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[196] + gpt_neox_layers_16_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[197] + gpt_neox_layers_16_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[198] + gpt_neox_layers_16_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[199] + gpt_neox_layers_16_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[200] + gpt_neox_layers_16_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[201] + gpt_neox_layers_16_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[202] + gpt_neox_layers_16_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[203] + gpt_neox_layers_16_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[204] + gpt_neox_layers_17_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[205] + gpt_neox_layers_17_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[206] + gpt_neox_layers_17_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[207] + gpt_neox_layers_17_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[208] + gpt_neox_layers_17_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[209] + gpt_neox_layers_17_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[210] + gpt_neox_layers_17_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[211] + gpt_neox_layers_17_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[212] + gpt_neox_layers_17_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[213] + gpt_neox_layers_17_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[214] + gpt_neox_layers_17_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[215] + gpt_neox_layers_17_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[216] + gpt_neox_layers_18_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[217] + gpt_neox_layers_18_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[218] + gpt_neox_layers_18_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[219] + gpt_neox_layers_18_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[220] + gpt_neox_layers_18_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[221] + gpt_neox_layers_18_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[222] + gpt_neox_layers_18_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[223] + gpt_neox_layers_18_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[224] + gpt_neox_layers_18_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[225] + gpt_neox_layers_18_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[226] + gpt_neox_layers_18_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[227] + gpt_neox_layers_18_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[228] + gpt_neox_layers_19_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[229] + gpt_neox_layers_19_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[230] + gpt_neox_layers_19_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[231] + gpt_neox_layers_19_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[232] + gpt_neox_layers_19_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[233] + gpt_neox_layers_19_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[234] + gpt_neox_layers_19_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[235] + gpt_neox_layers_19_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[236] + gpt_neox_layers_19_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[237] + gpt_neox_layers_19_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[238] + gpt_neox_layers_19_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[239] + gpt_neox_layers_19_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[240] + gpt_neox_layers_20_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[241] + gpt_neox_layers_20_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[242] + gpt_neox_layers_20_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[243] + gpt_neox_layers_20_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[244] + gpt_neox_layers_20_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[245] + gpt_neox_layers_20_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[246] + gpt_neox_layers_20_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[247] + gpt_neox_layers_20_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[248] + gpt_neox_layers_20_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[249] + gpt_neox_layers_20_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[250] + gpt_neox_layers_20_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[251] + gpt_neox_layers_20_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[252] + gpt_neox_layers_21_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[253] + gpt_neox_layers_21_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[254] + gpt_neox_layers_21_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[255] + gpt_neox_layers_21_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[256] + gpt_neox_layers_21_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[257] + gpt_neox_layers_21_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[258] + gpt_neox_layers_21_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[259] + gpt_neox_layers_21_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[260] + gpt_neox_layers_21_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[261] + gpt_neox_layers_21_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[262] + gpt_neox_layers_21_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[263] + gpt_neox_layers_21_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[264] + gpt_neox_layers_22_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[265] + gpt_neox_layers_22_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[266] + gpt_neox_layers_22_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[267] + gpt_neox_layers_22_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[268] + gpt_neox_layers_22_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[269] + gpt_neox_layers_22_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[270] + gpt_neox_layers_22_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[271] + gpt_neox_layers_22_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[272] + gpt_neox_layers_22_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[273] + gpt_neox_layers_22_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[274] + gpt_neox_layers_22_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[275] + gpt_neox_layers_22_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[276] + gpt_neox_layers_23_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[277] + gpt_neox_layers_23_input_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[278] + gpt_neox_layers_23_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[279] + gpt_neox_layers_23_post_attention_layernorm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[280] + gpt_neox_layers_23_attention_query_key_value_weight2: R.Tensor((6144, 2048), dtype="float16") = packed_params[281] + gpt_neox_layers_23_attention_query_key_value_bias2: R.Tensor((6144,), dtype="float16") = packed_params[282] + gpt_neox_layers_23_attention_dense_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[283] + gpt_neox_layers_23_attention_dense_bias2: R.Tensor((2048,), dtype="float16") = packed_params[284] + gpt_neox_layers_23_mlp_dense_h_to_4h_weight2: R.Tensor((8192, 2048), dtype="float16") = packed_params[285] + gpt_neox_layers_23_mlp_dense_h_to_4h_bias2: R.Tensor((8192,), dtype="float32") = packed_params[286] + gpt_neox_layers_23_mlp_dense_4h_to_h_weight2: R.Tensor((2048, 8192), dtype="float16") = packed_params[287] + gpt_neox_layers_23_mlp_dense_4h_to_h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[288] + gpt_neox_final_layer_norm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[289] + gpt_neox_final_layer_norm_bias2: R.Tensor((2048,), dtype="float16") = packed_params[290] + embed_out_weight2: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[291] + layer_norm49: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_input_layernorm_weight2, gpt_neox_layers_0_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims97: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_query_key_value_weight2, axes=None) + matmul97: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm49, permute_dims97, out_dtype="void") + add144: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul97, gpt_neox_layers_0_attention_query_key_value_bias2) + reshape96: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add144, R.shape([1, 1, 48, 128])) + reshape97: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape96, R.shape([1, 48, 128])) + lv25 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape97), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape98: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv25, R.shape([1, 1, 16, 128])) + reshape99: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape98, R.shape([1, 1, 2048])) + permute_dims98: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_dense_weight2, axes=None) + matmul98: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape99, permute_dims98, out_dtype="void") + add145: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul98, gpt_neox_layers_0_attention_dense_bias2) + layer_norm50: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight2, gpt_neox_layers_0_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims99: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, axes=None) + matmul99: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm50, permute_dims99, out_dtype="float32") + add146: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul99, gpt_neox_layers_0_mlp_dense_h_to_4h_bias2) + gelu24: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add146) + astype49: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu24, dtype="float16") + permute_dims100: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_4h_to_h_weight2, axes=None) + matmul100: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype49, permute_dims100, out_dtype="float32") + add147: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul100, gpt_neox_layers_0_mlp_dense_4h_to_h_bias2) + astype50: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add147, dtype="float16") + add148: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype50, add145) + add149: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add148, input_embed) + layer_norm51: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add149, gpt_neox_layers_1_input_layernorm_weight2, gpt_neox_layers_1_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims101: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_query_key_value_weight2, axes=None) + matmul101: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm51, permute_dims101, out_dtype="void") + add150: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul101, gpt_neox_layers_1_attention_query_key_value_bias2) + reshape100: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add150, R.shape([1, 1, 48, 128])) + reshape101: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape100, R.shape([1, 48, 128])) + lv26 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape101), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape102: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv26, R.shape([1, 1, 16, 128])) + reshape103: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape102, R.shape([1, 1, 2048])) + permute_dims102: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_dense_weight2, axes=None) + matmul102: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape103, permute_dims102, out_dtype="void") + add151: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul102, gpt_neox_layers_1_attention_dense_bias2) + layer_norm52: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add149, gpt_neox_layers_1_post_attention_layernorm_weight2, gpt_neox_layers_1_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims103: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_h_to_4h_weight2, axes=None) + matmul103: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm52, permute_dims103, out_dtype="float32") + add152: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul103, gpt_neox_layers_1_mlp_dense_h_to_4h_bias2) + gelu25: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add152) + astype51: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu25, dtype="float16") + permute_dims104: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_4h_to_h_weight2, axes=None) + matmul104: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype51, permute_dims104, out_dtype="float32") + add153: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul104, gpt_neox_layers_1_mlp_dense_4h_to_h_bias2) + astype52: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add153, dtype="float16") + add154: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype52, add151) + add155: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add154, add149) + layer_norm53: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add155, gpt_neox_layers_2_input_layernorm_weight2, gpt_neox_layers_2_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims105: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_query_key_value_weight2, axes=None) + matmul105: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm53, permute_dims105, out_dtype="void") + add156: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul105, gpt_neox_layers_2_attention_query_key_value_bias2) + reshape104: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add156, R.shape([1, 1, 48, 128])) + reshape105: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape104, R.shape([1, 48, 128])) + lv27 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape105), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape106: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv27, R.shape([1, 1, 16, 128])) + reshape107: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape106, R.shape([1, 1, 2048])) + permute_dims106: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_dense_weight2, axes=None) + matmul106: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape107, permute_dims106, out_dtype="void") + add157: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul106, gpt_neox_layers_2_attention_dense_bias2) + layer_norm54: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add155, gpt_neox_layers_2_post_attention_layernorm_weight2, gpt_neox_layers_2_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims107: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_h_to_4h_weight2, axes=None) + matmul107: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm54, permute_dims107, out_dtype="float32") + add158: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul107, gpt_neox_layers_2_mlp_dense_h_to_4h_bias2) + gelu26: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add158) + astype53: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu26, dtype="float16") + permute_dims108: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_4h_to_h_weight2, axes=None) + matmul108: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype53, permute_dims108, out_dtype="float32") + add159: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul108, gpt_neox_layers_2_mlp_dense_4h_to_h_bias2) + astype54: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add159, dtype="float16") + add160: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype54, add157) + add161: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add160, add155) + layer_norm55: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add161, gpt_neox_layers_3_input_layernorm_weight2, gpt_neox_layers_3_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims109: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_query_key_value_weight2, axes=None) + matmul109: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm55, permute_dims109, out_dtype="void") + add162: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul109, gpt_neox_layers_3_attention_query_key_value_bias2) + reshape108: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add162, R.shape([1, 1, 48, 128])) + reshape109: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape108, R.shape([1, 48, 128])) + lv28 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape109), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape110: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv28, R.shape([1, 1, 16, 128])) + reshape111: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape110, R.shape([1, 1, 2048])) + permute_dims110: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_dense_weight2, axes=None) + matmul110: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape111, permute_dims110, out_dtype="void") + add163: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul110, gpt_neox_layers_3_attention_dense_bias2) + layer_norm56: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add161, gpt_neox_layers_3_post_attention_layernorm_weight2, gpt_neox_layers_3_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims111: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_h_to_4h_weight2, axes=None) + matmul111: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm56, permute_dims111, out_dtype="float32") + add164: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul111, gpt_neox_layers_3_mlp_dense_h_to_4h_bias2) + gelu27: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add164) + astype55: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu27, dtype="float16") + permute_dims112: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_4h_to_h_weight2, axes=None) + matmul112: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype55, permute_dims112, out_dtype="float32") + add165: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul112, gpt_neox_layers_3_mlp_dense_4h_to_h_bias2) + astype56: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add165, dtype="float16") + add166: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype56, add163) + add167: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add166, add161) + layer_norm57: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add167, gpt_neox_layers_4_input_layernorm_weight2, gpt_neox_layers_4_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims113: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_query_key_value_weight2, axes=None) + matmul113: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm57, permute_dims113, out_dtype="void") + add168: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul113, gpt_neox_layers_4_attention_query_key_value_bias2) + reshape112: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add168, R.shape([1, 1, 48, 128])) + reshape113: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape112, R.shape([1, 48, 128])) + lv29 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape113), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape114: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv29, R.shape([1, 1, 16, 128])) + reshape115: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape114, R.shape([1, 1, 2048])) + permute_dims114: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_dense_weight2, axes=None) + matmul114: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape115, permute_dims114, out_dtype="void") + add169: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul114, gpt_neox_layers_4_attention_dense_bias2) + layer_norm58: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add167, gpt_neox_layers_4_post_attention_layernorm_weight2, gpt_neox_layers_4_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims115: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_h_to_4h_weight2, axes=None) + matmul115: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm58, permute_dims115, out_dtype="float32") + add170: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul115, gpt_neox_layers_4_mlp_dense_h_to_4h_bias2) + gelu28: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add170) + astype57: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu28, dtype="float16") + permute_dims116: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_4h_to_h_weight2, axes=None) + matmul116: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype57, permute_dims116, out_dtype="float32") + add171: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul116, gpt_neox_layers_4_mlp_dense_4h_to_h_bias2) + astype58: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add171, dtype="float16") + add172: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype58, add169) + add173: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add172, add167) + layer_norm59: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add173, gpt_neox_layers_5_input_layernorm_weight2, gpt_neox_layers_5_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims117: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_query_key_value_weight2, axes=None) + matmul117: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm59, permute_dims117, out_dtype="void") + add174: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul117, gpt_neox_layers_5_attention_query_key_value_bias2) + reshape116: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add174, R.shape([1, 1, 48, 128])) + reshape117: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape116, R.shape([1, 48, 128])) + lv30 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape117), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape118: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv30, R.shape([1, 1, 16, 128])) + reshape119: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape118, R.shape([1, 1, 2048])) + permute_dims118: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_dense_weight2, axes=None) + matmul118: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape119, permute_dims118, out_dtype="void") + add175: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul118, gpt_neox_layers_5_attention_dense_bias2) + layer_norm60: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add173, gpt_neox_layers_5_post_attention_layernorm_weight2, gpt_neox_layers_5_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims119: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_h_to_4h_weight2, axes=None) + matmul119: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm60, permute_dims119, out_dtype="float32") + add176: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul119, gpt_neox_layers_5_mlp_dense_h_to_4h_bias2) + gelu29: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add176) + astype59: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu29, dtype="float16") + permute_dims120: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_4h_to_h_weight2, axes=None) + matmul120: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype59, permute_dims120, out_dtype="float32") + add177: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul120, gpt_neox_layers_5_mlp_dense_4h_to_h_bias2) + astype60: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add177, dtype="float16") + add178: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype60, add175) + add179: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add178, add173) + layer_norm61: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add179, gpt_neox_layers_6_input_layernorm_weight2, gpt_neox_layers_6_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims121: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_query_key_value_weight2, axes=None) + matmul121: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm61, permute_dims121, out_dtype="void") + add180: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul121, gpt_neox_layers_6_attention_query_key_value_bias2) + reshape120: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add180, R.shape([1, 1, 48, 128])) + reshape121: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape120, R.shape([1, 48, 128])) + lv31 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape121), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape122: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv31, R.shape([1, 1, 16, 128])) + reshape123: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape122, R.shape([1, 1, 2048])) + permute_dims122: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_dense_weight2, axes=None) + matmul122: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape123, permute_dims122, out_dtype="void") + add181: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul122, gpt_neox_layers_6_attention_dense_bias2) + layer_norm62: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add179, gpt_neox_layers_6_post_attention_layernorm_weight2, gpt_neox_layers_6_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims123: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_h_to_4h_weight2, axes=None) + matmul123: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm62, permute_dims123, out_dtype="float32") + add182: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul123, gpt_neox_layers_6_mlp_dense_h_to_4h_bias2) + gelu30: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add182) + astype61: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu30, dtype="float16") + permute_dims124: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_4h_to_h_weight2, axes=None) + matmul124: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype61, permute_dims124, out_dtype="float32") + add183: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul124, gpt_neox_layers_6_mlp_dense_4h_to_h_bias2) + astype62: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add183, dtype="float16") + add184: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype62, add181) + add185: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add184, add179) + layer_norm63: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add185, gpt_neox_layers_7_input_layernorm_weight2, gpt_neox_layers_7_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims125: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_query_key_value_weight2, axes=None) + matmul125: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm63, permute_dims125, out_dtype="void") + add186: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul125, gpt_neox_layers_7_attention_query_key_value_bias2) + reshape124: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add186, R.shape([1, 1, 48, 128])) + reshape125: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape124, R.shape([1, 48, 128])) + lv32 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape125), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape126: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv32, R.shape([1, 1, 16, 128])) + reshape127: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape126, R.shape([1, 1, 2048])) + permute_dims126: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_dense_weight2, axes=None) + matmul126: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape127, permute_dims126, out_dtype="void") + add187: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul126, gpt_neox_layers_7_attention_dense_bias2) + layer_norm64: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add185, gpt_neox_layers_7_post_attention_layernorm_weight2, gpt_neox_layers_7_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims127: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_h_to_4h_weight2, axes=None) + matmul127: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm64, permute_dims127, out_dtype="float32") + add188: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul127, gpt_neox_layers_7_mlp_dense_h_to_4h_bias2) + gelu31: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add188) + astype63: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu31, dtype="float16") + permute_dims128: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_4h_to_h_weight2, axes=None) + matmul128: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype63, permute_dims128, out_dtype="float32") + add189: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul128, gpt_neox_layers_7_mlp_dense_4h_to_h_bias2) + astype64: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add189, dtype="float16") + add190: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype64, add187) + add191: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add190, add185) + layer_norm65: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add191, gpt_neox_layers_8_input_layernorm_weight2, gpt_neox_layers_8_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims129: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_query_key_value_weight2, axes=None) + matmul129: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm65, permute_dims129, out_dtype="void") + add192: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul129, gpt_neox_layers_8_attention_query_key_value_bias2) + reshape128: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add192, R.shape([1, 1, 48, 128])) + reshape129: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape128, R.shape([1, 48, 128])) + lv33 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape129), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape130: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv33, R.shape([1, 1, 16, 128])) + reshape131: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape130, R.shape([1, 1, 2048])) + permute_dims130: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_dense_weight2, axes=None) + matmul130: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape131, permute_dims130, out_dtype="void") + add193: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul130, gpt_neox_layers_8_attention_dense_bias2) + layer_norm66: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add191, gpt_neox_layers_8_post_attention_layernorm_weight2, gpt_neox_layers_8_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims131: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_h_to_4h_weight2, axes=None) + matmul131: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm66, permute_dims131, out_dtype="float32") + add194: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul131, gpt_neox_layers_8_mlp_dense_h_to_4h_bias2) + gelu32: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add194) + astype65: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu32, dtype="float16") + permute_dims132: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_4h_to_h_weight2, axes=None) + matmul132: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype65, permute_dims132, out_dtype="float32") + add195: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul132, gpt_neox_layers_8_mlp_dense_4h_to_h_bias2) + astype66: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add195, dtype="float16") + add196: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype66, add193) + add197: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add196, add191) + layer_norm67: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add197, gpt_neox_layers_9_input_layernorm_weight2, gpt_neox_layers_9_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims133: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_query_key_value_weight2, axes=None) + matmul133: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm67, permute_dims133, out_dtype="void") + add198: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul133, gpt_neox_layers_9_attention_query_key_value_bias2) + reshape132: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add198, R.shape([1, 1, 48, 128])) + reshape133: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape132, R.shape([1, 48, 128])) + lv34 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape133), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape134: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv34, R.shape([1, 1, 16, 128])) + reshape135: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape134, R.shape([1, 1, 2048])) + permute_dims134: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_dense_weight2, axes=None) + matmul134: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape135, permute_dims134, out_dtype="void") + add199: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul134, gpt_neox_layers_9_attention_dense_bias2) + layer_norm68: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add197, gpt_neox_layers_9_post_attention_layernorm_weight2, gpt_neox_layers_9_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims135: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_h_to_4h_weight2, axes=None) + matmul135: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm68, permute_dims135, out_dtype="float32") + add200: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul135, gpt_neox_layers_9_mlp_dense_h_to_4h_bias2) + gelu33: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add200) + astype67: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu33, dtype="float16") + permute_dims136: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_4h_to_h_weight2, axes=None) + matmul136: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype67, permute_dims136, out_dtype="float32") + add201: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul136, gpt_neox_layers_9_mlp_dense_4h_to_h_bias2) + astype68: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add201, dtype="float16") + add202: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype68, add199) + add203: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add202, add197) + layer_norm69: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add203, gpt_neox_layers_10_input_layernorm_weight2, gpt_neox_layers_10_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims137: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_query_key_value_weight2, axes=None) + matmul137: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm69, permute_dims137, out_dtype="void") + add204: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul137, gpt_neox_layers_10_attention_query_key_value_bias2) + reshape136: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add204, R.shape([1, 1, 48, 128])) + reshape137: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape136, R.shape([1, 48, 128])) + lv35 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape137), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape138: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv35, R.shape([1, 1, 16, 128])) + reshape139: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape138, R.shape([1, 1, 2048])) + permute_dims138: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_dense_weight2, axes=None) + matmul138: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape139, permute_dims138, out_dtype="void") + add205: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul138, gpt_neox_layers_10_attention_dense_bias2) + layer_norm70: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add203, gpt_neox_layers_10_post_attention_layernorm_weight2, gpt_neox_layers_10_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims139: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_h_to_4h_weight2, axes=None) + matmul139: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm70, permute_dims139, out_dtype="float32") + add206: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul139, gpt_neox_layers_10_mlp_dense_h_to_4h_bias2) + gelu34: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add206) + astype69: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu34, dtype="float16") + permute_dims140: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_4h_to_h_weight2, axes=None) + matmul140: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype69, permute_dims140, out_dtype="float32") + add207: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul140, gpt_neox_layers_10_mlp_dense_4h_to_h_bias2) + astype70: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add207, dtype="float16") + add208: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype70, add205) + add209: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add208, add203) + layer_norm71: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add209, gpt_neox_layers_11_input_layernorm_weight2, gpt_neox_layers_11_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims141: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_query_key_value_weight2, axes=None) + matmul141: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm71, permute_dims141, out_dtype="void") + add210: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul141, gpt_neox_layers_11_attention_query_key_value_bias2) + reshape140: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add210, R.shape([1, 1, 48, 128])) + reshape141: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape140, R.shape([1, 48, 128])) + lv36 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape141), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape142: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv36, R.shape([1, 1, 16, 128])) + reshape143: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape142, R.shape([1, 1, 2048])) + permute_dims142: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_dense_weight2, axes=None) + matmul142: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape143, permute_dims142, out_dtype="void") + add211: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul142, gpt_neox_layers_11_attention_dense_bias2) + layer_norm72: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add209, gpt_neox_layers_11_post_attention_layernorm_weight2, gpt_neox_layers_11_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims143: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_h_to_4h_weight2, axes=None) + matmul143: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm72, permute_dims143, out_dtype="float32") + add212: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul143, gpt_neox_layers_11_mlp_dense_h_to_4h_bias2) + gelu35: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add212) + astype71: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu35, dtype="float16") + permute_dims144: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_4h_to_h_weight2, axes=None) + matmul144: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype71, permute_dims144, out_dtype="float32") + add213: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul144, gpt_neox_layers_11_mlp_dense_4h_to_h_bias2) + astype72: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add213, dtype="float16") + add214: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype72, add211) + add215: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add214, add209) + layer_norm73: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add215, gpt_neox_layers_12_input_layernorm_weight2, gpt_neox_layers_12_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims145: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_query_key_value_weight2, axes=None) + matmul145: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm73, permute_dims145, out_dtype="void") + add216: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul145, gpt_neox_layers_12_attention_query_key_value_bias2) + reshape144: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add216, R.shape([1, 1, 48, 128])) + reshape145: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape144, R.shape([1, 48, 128])) + lv37 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape145), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape146: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv37, R.shape([1, 1, 16, 128])) + reshape147: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape146, R.shape([1, 1, 2048])) + permute_dims146: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_dense_weight2, axes=None) + matmul146: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape147, permute_dims146, out_dtype="void") + add217: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul146, gpt_neox_layers_12_attention_dense_bias2) + layer_norm74: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add215, gpt_neox_layers_12_post_attention_layernorm_weight2, gpt_neox_layers_12_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims147: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_h_to_4h_weight2, axes=None) + matmul147: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm74, permute_dims147, out_dtype="float32") + add218: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul147, gpt_neox_layers_12_mlp_dense_h_to_4h_bias2) + gelu36: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add218) + astype73: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu36, dtype="float16") + permute_dims148: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_4h_to_h_weight2, axes=None) + matmul148: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype73, permute_dims148, out_dtype="float32") + add219: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul148, gpt_neox_layers_12_mlp_dense_4h_to_h_bias2) + astype74: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add219, dtype="float16") + add220: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype74, add217) + add221: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add220, add215) + layer_norm75: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add221, gpt_neox_layers_13_input_layernorm_weight2, gpt_neox_layers_13_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims149: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_query_key_value_weight2, axes=None) + matmul149: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm75, permute_dims149, out_dtype="void") + add222: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul149, gpt_neox_layers_13_attention_query_key_value_bias2) + reshape148: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add222, R.shape([1, 1, 48, 128])) + reshape149: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape148, R.shape([1, 48, 128])) + lv38 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape149), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape150: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv38, R.shape([1, 1, 16, 128])) + reshape151: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape150, R.shape([1, 1, 2048])) + permute_dims150: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_dense_weight2, axes=None) + matmul150: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape151, permute_dims150, out_dtype="void") + add223: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul150, gpt_neox_layers_13_attention_dense_bias2) + layer_norm76: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add221, gpt_neox_layers_13_post_attention_layernorm_weight2, gpt_neox_layers_13_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims151: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_h_to_4h_weight2, axes=None) + matmul151: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm76, permute_dims151, out_dtype="float32") + add224: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul151, gpt_neox_layers_13_mlp_dense_h_to_4h_bias2) + gelu37: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add224) + astype75: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu37, dtype="float16") + permute_dims152: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_4h_to_h_weight2, axes=None) + matmul152: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype75, permute_dims152, out_dtype="float32") + add225: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul152, gpt_neox_layers_13_mlp_dense_4h_to_h_bias2) + astype76: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add225, dtype="float16") + add226: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype76, add223) + add227: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add226, add221) + layer_norm77: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add227, gpt_neox_layers_14_input_layernorm_weight2, gpt_neox_layers_14_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims153: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_query_key_value_weight2, axes=None) + matmul153: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm77, permute_dims153, out_dtype="void") + add228: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul153, gpt_neox_layers_14_attention_query_key_value_bias2) + reshape152: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add228, R.shape([1, 1, 48, 128])) + reshape153: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape152, R.shape([1, 48, 128])) + lv39 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape153), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape154: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv39, R.shape([1, 1, 16, 128])) + reshape155: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape154, R.shape([1, 1, 2048])) + permute_dims154: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_dense_weight2, axes=None) + matmul154: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape155, permute_dims154, out_dtype="void") + add229: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul154, gpt_neox_layers_14_attention_dense_bias2) + layer_norm78: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add227, gpt_neox_layers_14_post_attention_layernorm_weight2, gpt_neox_layers_14_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims155: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_h_to_4h_weight2, axes=None) + matmul155: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm78, permute_dims155, out_dtype="float32") + add230: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul155, gpt_neox_layers_14_mlp_dense_h_to_4h_bias2) + gelu38: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add230) + astype77: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu38, dtype="float16") + permute_dims156: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_4h_to_h_weight2, axes=None) + matmul156: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype77, permute_dims156, out_dtype="float32") + add231: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul156, gpt_neox_layers_14_mlp_dense_4h_to_h_bias2) + astype78: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add231, dtype="float16") + add232: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype78, add229) + add233: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add232, add227) + layer_norm79: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add233, gpt_neox_layers_15_input_layernorm_weight2, gpt_neox_layers_15_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims157: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_query_key_value_weight2, axes=None) + matmul157: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm79, permute_dims157, out_dtype="void") + add234: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul157, gpt_neox_layers_15_attention_query_key_value_bias2) + reshape156: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add234, R.shape([1, 1, 48, 128])) + reshape157: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape156, R.shape([1, 48, 128])) + lv40 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape157), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape158: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv40, R.shape([1, 1, 16, 128])) + reshape159: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape158, R.shape([1, 1, 2048])) + permute_dims158: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_dense_weight2, axes=None) + matmul158: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape159, permute_dims158, out_dtype="void") + add235: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul158, gpt_neox_layers_15_attention_dense_bias2) + layer_norm80: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add233, gpt_neox_layers_15_post_attention_layernorm_weight2, gpt_neox_layers_15_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims159: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_h_to_4h_weight2, axes=None) + matmul159: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm80, permute_dims159, out_dtype="float32") + add236: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul159, gpt_neox_layers_15_mlp_dense_h_to_4h_bias2) + gelu39: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add236) + astype79: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu39, dtype="float16") + permute_dims160: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_4h_to_h_weight2, axes=None) + matmul160: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype79, permute_dims160, out_dtype="float32") + add237: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul160, gpt_neox_layers_15_mlp_dense_4h_to_h_bias2) + astype80: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add237, dtype="float16") + add238: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype80, add235) + add239: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add238, add233) + layer_norm81: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add239, gpt_neox_layers_16_input_layernorm_weight2, gpt_neox_layers_16_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims161: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_query_key_value_weight2, axes=None) + matmul161: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm81, permute_dims161, out_dtype="void") + add240: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul161, gpt_neox_layers_16_attention_query_key_value_bias2) + reshape160: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add240, R.shape([1, 1, 48, 128])) + reshape161: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape160, R.shape([1, 48, 128])) + lv41 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape161), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape162: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv41, R.shape([1, 1, 16, 128])) + reshape163: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape162, R.shape([1, 1, 2048])) + permute_dims162: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_dense_weight2, axes=None) + matmul162: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape163, permute_dims162, out_dtype="void") + add241: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul162, gpt_neox_layers_16_attention_dense_bias2) + layer_norm82: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add239, gpt_neox_layers_16_post_attention_layernorm_weight2, gpt_neox_layers_16_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims163: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_h_to_4h_weight2, axes=None) + matmul163: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm82, permute_dims163, out_dtype="float32") + add242: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul163, gpt_neox_layers_16_mlp_dense_h_to_4h_bias2) + gelu40: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add242) + astype81: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu40, dtype="float16") + permute_dims164: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_4h_to_h_weight2, axes=None) + matmul164: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype81, permute_dims164, out_dtype="float32") + add243: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul164, gpt_neox_layers_16_mlp_dense_4h_to_h_bias2) + astype82: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add243, dtype="float16") + add244: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype82, add241) + add245: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add244, add239) + layer_norm83: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add245, gpt_neox_layers_17_input_layernorm_weight2, gpt_neox_layers_17_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims165: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_query_key_value_weight2, axes=None) + matmul165: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm83, permute_dims165, out_dtype="void") + add246: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul165, gpt_neox_layers_17_attention_query_key_value_bias2) + reshape164: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add246, R.shape([1, 1, 48, 128])) + reshape165: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape164, R.shape([1, 48, 128])) + lv42 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape165), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape166: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv42, R.shape([1, 1, 16, 128])) + reshape167: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape166, R.shape([1, 1, 2048])) + permute_dims166: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_dense_weight2, axes=None) + matmul166: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape167, permute_dims166, out_dtype="void") + add247: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul166, gpt_neox_layers_17_attention_dense_bias2) + layer_norm84: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add245, gpt_neox_layers_17_post_attention_layernorm_weight2, gpt_neox_layers_17_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims167: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_h_to_4h_weight2, axes=None) + matmul167: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm84, permute_dims167, out_dtype="float32") + add248: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul167, gpt_neox_layers_17_mlp_dense_h_to_4h_bias2) + gelu41: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add248) + astype83: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu41, dtype="float16") + permute_dims168: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_4h_to_h_weight2, axes=None) + matmul168: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype83, permute_dims168, out_dtype="float32") + add249: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul168, gpt_neox_layers_17_mlp_dense_4h_to_h_bias2) + astype84: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add249, dtype="float16") + add250: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype84, add247) + add251: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add250, add245) + layer_norm85: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add251, gpt_neox_layers_18_input_layernorm_weight2, gpt_neox_layers_18_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims169: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_query_key_value_weight2, axes=None) + matmul169: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm85, permute_dims169, out_dtype="void") + add252: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul169, gpt_neox_layers_18_attention_query_key_value_bias2) + reshape168: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add252, R.shape([1, 1, 48, 128])) + reshape169: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape168, R.shape([1, 48, 128])) + lv43 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape169), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape170: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv43, R.shape([1, 1, 16, 128])) + reshape171: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape170, R.shape([1, 1, 2048])) + permute_dims170: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_dense_weight2, axes=None) + matmul170: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape171, permute_dims170, out_dtype="void") + add253: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul170, gpt_neox_layers_18_attention_dense_bias2) + layer_norm86: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add251, gpt_neox_layers_18_post_attention_layernorm_weight2, gpt_neox_layers_18_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims171: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_h_to_4h_weight2, axes=None) + matmul171: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm86, permute_dims171, out_dtype="float32") + add254: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul171, gpt_neox_layers_18_mlp_dense_h_to_4h_bias2) + gelu42: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add254) + astype85: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu42, dtype="float16") + permute_dims172: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_4h_to_h_weight2, axes=None) + matmul172: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype85, permute_dims172, out_dtype="float32") + add255: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul172, gpt_neox_layers_18_mlp_dense_4h_to_h_bias2) + astype86: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add255, dtype="float16") + add256: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype86, add253) + add257: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add256, add251) + layer_norm87: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add257, gpt_neox_layers_19_input_layernorm_weight2, gpt_neox_layers_19_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims173: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_query_key_value_weight2, axes=None) + matmul173: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm87, permute_dims173, out_dtype="void") + add258: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul173, gpt_neox_layers_19_attention_query_key_value_bias2) + reshape172: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add258, R.shape([1, 1, 48, 128])) + reshape173: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape172, R.shape([1, 48, 128])) + lv44 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape173), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape174: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv44, R.shape([1, 1, 16, 128])) + reshape175: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape174, R.shape([1, 1, 2048])) + permute_dims174: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_dense_weight2, axes=None) + matmul174: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape175, permute_dims174, out_dtype="void") + add259: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul174, gpt_neox_layers_19_attention_dense_bias2) + layer_norm88: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add257, gpt_neox_layers_19_post_attention_layernorm_weight2, gpt_neox_layers_19_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims175: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_h_to_4h_weight2, axes=None) + matmul175: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm88, permute_dims175, out_dtype="float32") + add260: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul175, gpt_neox_layers_19_mlp_dense_h_to_4h_bias2) + gelu43: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add260) + astype87: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu43, dtype="float16") + permute_dims176: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_4h_to_h_weight2, axes=None) + matmul176: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype87, permute_dims176, out_dtype="float32") + add261: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul176, gpt_neox_layers_19_mlp_dense_4h_to_h_bias2) + astype88: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add261, dtype="float16") + add262: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype88, add259) + add263: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add262, add257) + layer_norm89: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add263, gpt_neox_layers_20_input_layernorm_weight2, gpt_neox_layers_20_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims177: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_query_key_value_weight2, axes=None) + matmul177: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm89, permute_dims177, out_dtype="void") + add264: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul177, gpt_neox_layers_20_attention_query_key_value_bias2) + reshape176: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add264, R.shape([1, 1, 48, 128])) + reshape177: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape176, R.shape([1, 48, 128])) + lv45 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape177), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape178: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv45, R.shape([1, 1, 16, 128])) + reshape179: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape178, R.shape([1, 1, 2048])) + permute_dims178: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_dense_weight2, axes=None) + matmul178: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape179, permute_dims178, out_dtype="void") + add265: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul178, gpt_neox_layers_20_attention_dense_bias2) + layer_norm90: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add263, gpt_neox_layers_20_post_attention_layernorm_weight2, gpt_neox_layers_20_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims179: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_h_to_4h_weight2, axes=None) + matmul179: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm90, permute_dims179, out_dtype="float32") + add266: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul179, gpt_neox_layers_20_mlp_dense_h_to_4h_bias2) + gelu44: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add266) + astype89: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu44, dtype="float16") + permute_dims180: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_4h_to_h_weight2, axes=None) + matmul180: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype89, permute_dims180, out_dtype="float32") + add267: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul180, gpt_neox_layers_20_mlp_dense_4h_to_h_bias2) + astype90: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add267, dtype="float16") + add268: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype90, add265) + add269: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add268, add263) + layer_norm91: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add269, gpt_neox_layers_21_input_layernorm_weight2, gpt_neox_layers_21_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims181: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_query_key_value_weight2, axes=None) + matmul181: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm91, permute_dims181, out_dtype="void") + add270: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul181, gpt_neox_layers_21_attention_query_key_value_bias2) + reshape180: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add270, R.shape([1, 1, 48, 128])) + reshape181: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape180, R.shape([1, 48, 128])) + lv46 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape181), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape182: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv46, R.shape([1, 1, 16, 128])) + reshape183: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape182, R.shape([1, 1, 2048])) + permute_dims182: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_dense_weight2, axes=None) + matmul182: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape183, permute_dims182, out_dtype="void") + add271: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul182, gpt_neox_layers_21_attention_dense_bias2) + layer_norm92: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add269, gpt_neox_layers_21_post_attention_layernorm_weight2, gpt_neox_layers_21_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims183: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_h_to_4h_weight2, axes=None) + matmul183: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm92, permute_dims183, out_dtype="float32") + add272: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul183, gpt_neox_layers_21_mlp_dense_h_to_4h_bias2) + gelu45: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add272) + astype91: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu45, dtype="float16") + permute_dims184: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_4h_to_h_weight2, axes=None) + matmul184: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype91, permute_dims184, out_dtype="float32") + add273: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul184, gpt_neox_layers_21_mlp_dense_4h_to_h_bias2) + astype92: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add273, dtype="float16") + add274: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype92, add271) + add275: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add274, add269) + layer_norm93: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add275, gpt_neox_layers_22_input_layernorm_weight2, gpt_neox_layers_22_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims185: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_query_key_value_weight2, axes=None) + matmul185: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm93, permute_dims185, out_dtype="void") + add276: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul185, gpt_neox_layers_22_attention_query_key_value_bias2) + reshape184: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add276, R.shape([1, 1, 48, 128])) + reshape185: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape184, R.shape([1, 48, 128])) + lv47 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape185), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape186: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv47, R.shape([1, 1, 16, 128])) + reshape187: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape186, R.shape([1, 1, 2048])) + permute_dims186: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_dense_weight2, axes=None) + matmul186: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape187, permute_dims186, out_dtype="void") + add277: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul186, gpt_neox_layers_22_attention_dense_bias2) + layer_norm94: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add275, gpt_neox_layers_22_post_attention_layernorm_weight2, gpt_neox_layers_22_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims187: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_h_to_4h_weight2, axes=None) + matmul187: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm94, permute_dims187, out_dtype="float32") + add278: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul187, gpt_neox_layers_22_mlp_dense_h_to_4h_bias2) + gelu46: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add278) + astype93: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu46, dtype="float16") + permute_dims188: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_4h_to_h_weight2, axes=None) + matmul188: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype93, permute_dims188, out_dtype="float32") + add279: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul188, gpt_neox_layers_22_mlp_dense_4h_to_h_bias2) + astype94: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add279, dtype="float16") + add280: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype94, add277) + add281: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add280, add275) + layer_norm95: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add281, gpt_neox_layers_23_input_layernorm_weight2, gpt_neox_layers_23_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims189: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_query_key_value_weight2, axes=None) + matmul189: R.Tensor((1, 1, 6144), dtype="float16") = R.matmul(layer_norm95, permute_dims189, out_dtype="void") + add282: R.Tensor((1, 1, 6144), dtype="float16") = R.add(matmul189, gpt_neox_layers_23_attention_query_key_value_bias2) + reshape188: R.Tensor((1, 1, 48, 128), dtype="float16") = R.reshape(add282, R.shape([1, 1, 48, 128])) + reshape189: R.Tensor((1, 48, 128), dtype="float16") = R.reshape(reshape188, R.shape([1, 48, 128])) + lv48 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape189), out_sinfo=R.Tensor((1, 16, 128), dtype="float16")) + reshape190: R.Tensor((1, 1, 16, 128), dtype="float16") = R.reshape(lv48, R.shape([1, 1, 16, 128])) + reshape191: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape190, R.shape([1, 1, 2048])) + permute_dims190: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_dense_weight2, axes=None) + matmul190: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape191, permute_dims190, out_dtype="void") + add283: R.Tensor((1, 1, 2048), dtype="float16") = R.add(matmul190, gpt_neox_layers_23_attention_dense_bias2) + layer_norm96: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add281, gpt_neox_layers_23_post_attention_layernorm_weight2, gpt_neox_layers_23_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims191: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_h_to_4h_weight2, axes=None) + matmul191: R.Tensor((1, 1, 8192), dtype="float32") = R.matmul(layer_norm96, permute_dims191, out_dtype="float32") + add284: R.Tensor((1, 1, 8192), dtype="float32") = R.add(matmul191, gpt_neox_layers_23_mlp_dense_h_to_4h_bias2) + gelu47: R.Tensor((1, 1, 8192), dtype="float32") = R.nn.gelu(add284) + astype95: R.Tensor((1, 1, 8192), dtype="float16") = R.astype(gelu47, dtype="float16") + permute_dims192: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_4h_to_h_weight2, axes=None) + matmul192: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(astype95, permute_dims192, out_dtype="float32") + add285: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul192, gpt_neox_layers_23_mlp_dense_4h_to_h_bias2) + astype96: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(add285, dtype="float16") + add286: R.Tensor((1, 1, 2048), dtype="float16") = R.add(astype96, add283) + add287: R.Tensor((1, 1, 2048), dtype="float16") = R.add(add286, add281) + layer_norm97: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.layer_norm(add287, gpt_neox_final_layer_norm_weight2, gpt_neox_final_layer_norm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims193: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(embed_out_weight2, axes=None) + matmul193: R.Tensor((1, 1, vocab_size), dtype="float16") = R.matmul(layer_norm97, permute_dims193, out_dtype="void") + astype97: R.Tensor((1, 1, vocab_size), dtype="float32") = R.astype(matmul193, dtype="float32") + gv2: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = astype97, paged_kv_cache + R.output(gv2) + return gv2 + + @R.function + def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor(("vocab_size", 2048), dtype="float16"))) -> R.Tensor(("seq_len", 2048), dtype="float16"): + seq_len = T.int64() + vocab_size = T.int64() + R.func_attr({"num_input": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + with R.dataflow(): + gpt_neox_embed_in_weight: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] + gpt_neox_layers_0_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[1] + gpt_neox_layers_0_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[2] + gpt_neox_layers_0_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[3] + gpt_neox_layers_0_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[4] + gpt_neox_layers_0_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[5] + gpt_neox_layers_0_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[6] + gpt_neox_layers_0_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[7] + gpt_neox_layers_0_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[10] + gpt_neox_layers_0_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[11] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[12] + gpt_neox_layers_1_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[13] + gpt_neox_layers_1_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[14] + gpt_neox_layers_1_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[15] + gpt_neox_layers_1_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[16] + gpt_neox_layers_1_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[17] + gpt_neox_layers_1_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[18] + gpt_neox_layers_1_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[19] + gpt_neox_layers_1_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[20] + gpt_neox_layers_1_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[21] + gpt_neox_layers_1_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[22] + gpt_neox_layers_1_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[23] + gpt_neox_layers_1_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[24] + gpt_neox_layers_2_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[25] + gpt_neox_layers_2_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[26] + gpt_neox_layers_2_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[27] + gpt_neox_layers_2_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[28] + gpt_neox_layers_2_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[29] + gpt_neox_layers_2_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[30] + gpt_neox_layers_2_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[31] + gpt_neox_layers_2_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[32] + gpt_neox_layers_2_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[33] + gpt_neox_layers_2_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[34] + gpt_neox_layers_2_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[35] + gpt_neox_layers_2_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[36] + gpt_neox_layers_3_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[37] + gpt_neox_layers_3_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[38] + gpt_neox_layers_3_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[39] + gpt_neox_layers_3_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[40] + gpt_neox_layers_3_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[41] + gpt_neox_layers_3_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[42] + gpt_neox_layers_3_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[43] + gpt_neox_layers_3_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[44] + gpt_neox_layers_3_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[45] + gpt_neox_layers_3_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[46] + gpt_neox_layers_3_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[47] + gpt_neox_layers_3_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[48] + gpt_neox_layers_4_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[49] + gpt_neox_layers_4_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[50] + gpt_neox_layers_4_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[51] + gpt_neox_layers_4_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[52] + gpt_neox_layers_4_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[53] + gpt_neox_layers_4_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[54] + gpt_neox_layers_4_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[55] + gpt_neox_layers_4_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[56] + gpt_neox_layers_4_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[57] + gpt_neox_layers_4_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[58] + gpt_neox_layers_4_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[59] + gpt_neox_layers_4_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[60] + gpt_neox_layers_5_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[61] + gpt_neox_layers_5_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[62] + gpt_neox_layers_5_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[63] + gpt_neox_layers_5_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[64] + gpt_neox_layers_5_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[65] + gpt_neox_layers_5_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[66] + gpt_neox_layers_5_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[67] + gpt_neox_layers_5_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[68] + gpt_neox_layers_5_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[69] + gpt_neox_layers_5_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[70] + gpt_neox_layers_5_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[71] + gpt_neox_layers_5_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[72] + gpt_neox_layers_6_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[73] + gpt_neox_layers_6_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[74] + gpt_neox_layers_6_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[75] + gpt_neox_layers_6_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[76] + gpt_neox_layers_6_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[77] + gpt_neox_layers_6_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[78] + gpt_neox_layers_6_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[79] + gpt_neox_layers_6_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[80] + gpt_neox_layers_6_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[81] + gpt_neox_layers_6_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[82] + gpt_neox_layers_6_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[83] + gpt_neox_layers_6_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[84] + gpt_neox_layers_7_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[85] + gpt_neox_layers_7_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[86] + gpt_neox_layers_7_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[87] + gpt_neox_layers_7_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[88] + gpt_neox_layers_7_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[89] + gpt_neox_layers_7_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[90] + gpt_neox_layers_7_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[91] + gpt_neox_layers_7_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[92] + gpt_neox_layers_7_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[93] + gpt_neox_layers_7_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[94] + gpt_neox_layers_7_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[95] + gpt_neox_layers_7_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[96] + gpt_neox_layers_8_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[97] + gpt_neox_layers_8_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[98] + gpt_neox_layers_8_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[99] + gpt_neox_layers_8_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[100] + gpt_neox_layers_8_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[101] + gpt_neox_layers_8_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[102] + gpt_neox_layers_8_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[103] + gpt_neox_layers_8_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[104] + gpt_neox_layers_8_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[105] + gpt_neox_layers_8_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[106] + gpt_neox_layers_8_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[107] + gpt_neox_layers_8_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[108] + gpt_neox_layers_9_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[109] + gpt_neox_layers_9_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[110] + gpt_neox_layers_9_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[111] + gpt_neox_layers_9_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[112] + gpt_neox_layers_9_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[113] + gpt_neox_layers_9_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[114] + gpt_neox_layers_9_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[115] + gpt_neox_layers_9_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[116] + gpt_neox_layers_9_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[117] + gpt_neox_layers_9_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[118] + gpt_neox_layers_9_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[119] + gpt_neox_layers_9_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[120] + gpt_neox_layers_10_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[121] + gpt_neox_layers_10_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[122] + gpt_neox_layers_10_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[123] + gpt_neox_layers_10_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[124] + gpt_neox_layers_10_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[125] + gpt_neox_layers_10_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[126] + gpt_neox_layers_10_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[127] + gpt_neox_layers_10_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[128] + gpt_neox_layers_10_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[129] + gpt_neox_layers_10_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[130] + gpt_neox_layers_10_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[131] + gpt_neox_layers_10_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[132] + gpt_neox_layers_11_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[133] + gpt_neox_layers_11_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[134] + gpt_neox_layers_11_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[135] + gpt_neox_layers_11_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[136] + gpt_neox_layers_11_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[137] + gpt_neox_layers_11_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[138] + gpt_neox_layers_11_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[139] + gpt_neox_layers_11_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[140] + gpt_neox_layers_11_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[141] + gpt_neox_layers_11_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[142] + gpt_neox_layers_11_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[143] + gpt_neox_layers_11_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[144] + gpt_neox_layers_12_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[145] + gpt_neox_layers_12_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[146] + gpt_neox_layers_12_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[147] + gpt_neox_layers_12_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[148] + gpt_neox_layers_12_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[149] + gpt_neox_layers_12_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[150] + gpt_neox_layers_12_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[151] + gpt_neox_layers_12_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[152] + gpt_neox_layers_12_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[153] + gpt_neox_layers_12_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[154] + gpt_neox_layers_12_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[155] + gpt_neox_layers_12_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[156] + gpt_neox_layers_13_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[157] + gpt_neox_layers_13_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[158] + gpt_neox_layers_13_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[159] + gpt_neox_layers_13_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[160] + gpt_neox_layers_13_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[161] + gpt_neox_layers_13_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[162] + gpt_neox_layers_13_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[163] + gpt_neox_layers_13_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[164] + gpt_neox_layers_13_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[165] + gpt_neox_layers_13_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[166] + gpt_neox_layers_13_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[167] + gpt_neox_layers_13_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[168] + gpt_neox_layers_14_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[169] + gpt_neox_layers_14_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[170] + gpt_neox_layers_14_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[171] + gpt_neox_layers_14_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[172] + gpt_neox_layers_14_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[173] + gpt_neox_layers_14_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[174] + gpt_neox_layers_14_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[175] + gpt_neox_layers_14_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[176] + gpt_neox_layers_14_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[177] + gpt_neox_layers_14_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[178] + gpt_neox_layers_14_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[179] + gpt_neox_layers_14_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[180] + gpt_neox_layers_15_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[181] + gpt_neox_layers_15_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[182] + gpt_neox_layers_15_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[183] + gpt_neox_layers_15_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[184] + gpt_neox_layers_15_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[185] + gpt_neox_layers_15_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[186] + gpt_neox_layers_15_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[187] + gpt_neox_layers_15_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[188] + gpt_neox_layers_15_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[189] + gpt_neox_layers_15_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[190] + gpt_neox_layers_15_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[191] + gpt_neox_layers_15_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[192] + gpt_neox_layers_16_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[193] + gpt_neox_layers_16_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[194] + gpt_neox_layers_16_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[195] + gpt_neox_layers_16_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[196] + gpt_neox_layers_16_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[197] + gpt_neox_layers_16_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[198] + gpt_neox_layers_16_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[199] + gpt_neox_layers_16_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[200] + gpt_neox_layers_16_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[201] + gpt_neox_layers_16_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[202] + gpt_neox_layers_16_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[203] + gpt_neox_layers_16_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[204] + gpt_neox_layers_17_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[205] + gpt_neox_layers_17_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[206] + gpt_neox_layers_17_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[207] + gpt_neox_layers_17_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[208] + gpt_neox_layers_17_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[209] + gpt_neox_layers_17_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[210] + gpt_neox_layers_17_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[211] + gpt_neox_layers_17_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[212] + gpt_neox_layers_17_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[213] + gpt_neox_layers_17_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[214] + gpt_neox_layers_17_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[215] + gpt_neox_layers_17_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[216] + gpt_neox_layers_18_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[217] + gpt_neox_layers_18_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[218] + gpt_neox_layers_18_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[219] + gpt_neox_layers_18_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[220] + gpt_neox_layers_18_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[221] + gpt_neox_layers_18_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[222] + gpt_neox_layers_18_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[223] + gpt_neox_layers_18_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[224] + gpt_neox_layers_18_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[225] + gpt_neox_layers_18_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[226] + gpt_neox_layers_18_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[227] + gpt_neox_layers_18_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[228] + gpt_neox_layers_19_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[229] + gpt_neox_layers_19_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[230] + gpt_neox_layers_19_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[231] + gpt_neox_layers_19_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[232] + gpt_neox_layers_19_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[233] + gpt_neox_layers_19_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[234] + gpt_neox_layers_19_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[235] + gpt_neox_layers_19_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[236] + gpt_neox_layers_19_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[237] + gpt_neox_layers_19_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[238] + gpt_neox_layers_19_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[239] + gpt_neox_layers_19_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[240] + gpt_neox_layers_20_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[241] + gpt_neox_layers_20_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[242] + gpt_neox_layers_20_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[243] + gpt_neox_layers_20_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[244] + gpt_neox_layers_20_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[245] + gpt_neox_layers_20_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[246] + gpt_neox_layers_20_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[247] + gpt_neox_layers_20_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[248] + gpt_neox_layers_20_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[249] + gpt_neox_layers_20_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[250] + gpt_neox_layers_20_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[251] + gpt_neox_layers_20_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[252] + gpt_neox_layers_21_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[253] + gpt_neox_layers_21_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[254] + gpt_neox_layers_21_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[255] + gpt_neox_layers_21_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[256] + gpt_neox_layers_21_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[257] + gpt_neox_layers_21_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[258] + gpt_neox_layers_21_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[259] + gpt_neox_layers_21_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[260] + gpt_neox_layers_21_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[261] + gpt_neox_layers_21_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[262] + gpt_neox_layers_21_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[263] + gpt_neox_layers_21_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[264] + gpt_neox_layers_22_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[265] + gpt_neox_layers_22_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[266] + gpt_neox_layers_22_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[267] + gpt_neox_layers_22_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[268] + gpt_neox_layers_22_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[269] + gpt_neox_layers_22_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[270] + gpt_neox_layers_22_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[271] + gpt_neox_layers_22_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[272] + gpt_neox_layers_22_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[273] + gpt_neox_layers_22_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[274] + gpt_neox_layers_22_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[275] + gpt_neox_layers_22_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[276] + gpt_neox_layers_23_input_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[277] + gpt_neox_layers_23_input_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[278] + gpt_neox_layers_23_post_attention_layernorm_weight: R.Tensor((2048,), dtype="float16") = packed_params[279] + gpt_neox_layers_23_post_attention_layernorm_bias: R.Tensor((2048,), dtype="float16") = packed_params[280] + gpt_neox_layers_23_attention_query_key_value_weight: R.Tensor((6144, 2048), dtype="float16") = packed_params[281] + gpt_neox_layers_23_attention_query_key_value_bias: R.Tensor((6144,), dtype="float16") = packed_params[282] + gpt_neox_layers_23_attention_dense_weight: R.Tensor((2048, 2048), dtype="float16") = packed_params[283] + gpt_neox_layers_23_attention_dense_bias: R.Tensor((2048,), dtype="float16") = packed_params[284] + gpt_neox_layers_23_mlp_dense_h_to_4h_weight: R.Tensor((8192, 2048), dtype="float16") = packed_params[285] + gpt_neox_layers_23_mlp_dense_h_to_4h_bias: R.Tensor((8192,), dtype="float32") = packed_params[286] + gpt_neox_layers_23_mlp_dense_4h_to_h_weight: R.Tensor((2048, 8192), dtype="float16") = packed_params[287] + gpt_neox_layers_23_mlp_dense_4h_to_h_bias: R.Tensor((2048,), dtype="float32") = packed_params[288] + gpt_neox_final_layer_norm_weight: R.Tensor((2048,), dtype="float16") = packed_params[289] + gpt_neox_final_layer_norm_bias: R.Tensor((2048,), dtype="float16") = packed_params[290] + embed_out_weight: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[291] + take: R.Tensor((seq_len, 2048), dtype="float16") = R.take(gpt_neox_embed_in_weight, input_ids, axis=0) + gv: R.Tensor((seq_len, 2048), dtype="float16") = take + R.output(gv) + return gv + + @R.function + def prefill(input_embed: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((6144, 2048), dtype="float16"), R.Tensor((6144,), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((8192, 2048), dtype="float16"), R.Tensor((8192,), dtype="float32"), R.Tensor((2048, 8192), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor(("vocab_size", 2048), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): + vocab_size = T.int64() + seq_len = T.int64() + R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + cls = Module + with R.dataflow(): + gpt_neox_embed_in_weight1: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] + gpt_neox_layers_0_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[1] + gpt_neox_layers_0_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[2] + gpt_neox_layers_0_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[3] + gpt_neox_layers_0_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[4] + gpt_neox_layers_0_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[5] + gpt_neox_layers_0_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[6] + gpt_neox_layers_0_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[7] + gpt_neox_layers_0_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[10] + gpt_neox_layers_0_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[11] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[12] + gpt_neox_layers_1_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[13] + gpt_neox_layers_1_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[14] + gpt_neox_layers_1_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[15] + gpt_neox_layers_1_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[16] + gpt_neox_layers_1_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[17] + gpt_neox_layers_1_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[18] + gpt_neox_layers_1_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[19] + gpt_neox_layers_1_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[20] + gpt_neox_layers_1_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[21] + gpt_neox_layers_1_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[22] + gpt_neox_layers_1_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[23] + gpt_neox_layers_1_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[24] + gpt_neox_layers_2_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[25] + gpt_neox_layers_2_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[26] + gpt_neox_layers_2_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[27] + gpt_neox_layers_2_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[28] + gpt_neox_layers_2_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[29] + gpt_neox_layers_2_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[30] + gpt_neox_layers_2_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[31] + gpt_neox_layers_2_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[32] + gpt_neox_layers_2_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[33] + gpt_neox_layers_2_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[34] + gpt_neox_layers_2_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[35] + gpt_neox_layers_2_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[36] + gpt_neox_layers_3_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[37] + gpt_neox_layers_3_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[38] + gpt_neox_layers_3_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[39] + gpt_neox_layers_3_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[40] + gpt_neox_layers_3_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[41] + gpt_neox_layers_3_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[42] + gpt_neox_layers_3_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[43] + gpt_neox_layers_3_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[44] + gpt_neox_layers_3_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[45] + gpt_neox_layers_3_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[46] + gpt_neox_layers_3_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[47] + gpt_neox_layers_3_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[48] + gpt_neox_layers_4_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[49] + gpt_neox_layers_4_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[50] + gpt_neox_layers_4_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[51] + gpt_neox_layers_4_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[52] + gpt_neox_layers_4_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[53] + gpt_neox_layers_4_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[54] + gpt_neox_layers_4_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[55] + gpt_neox_layers_4_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[56] + gpt_neox_layers_4_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[57] + gpt_neox_layers_4_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[58] + gpt_neox_layers_4_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[59] + gpt_neox_layers_4_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[60] + gpt_neox_layers_5_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[61] + gpt_neox_layers_5_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[62] + gpt_neox_layers_5_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[63] + gpt_neox_layers_5_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[64] + gpt_neox_layers_5_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[65] + gpt_neox_layers_5_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[66] + gpt_neox_layers_5_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[67] + gpt_neox_layers_5_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[68] + gpt_neox_layers_5_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[69] + gpt_neox_layers_5_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[70] + gpt_neox_layers_5_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[71] + gpt_neox_layers_5_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[72] + gpt_neox_layers_6_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[73] + gpt_neox_layers_6_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[74] + gpt_neox_layers_6_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[75] + gpt_neox_layers_6_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[76] + gpt_neox_layers_6_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[77] + gpt_neox_layers_6_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[78] + gpt_neox_layers_6_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[79] + gpt_neox_layers_6_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[80] + gpt_neox_layers_6_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[81] + gpt_neox_layers_6_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[82] + gpt_neox_layers_6_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[83] + gpt_neox_layers_6_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[84] + gpt_neox_layers_7_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[85] + gpt_neox_layers_7_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[86] + gpt_neox_layers_7_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[87] + gpt_neox_layers_7_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[88] + gpt_neox_layers_7_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[89] + gpt_neox_layers_7_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[90] + gpt_neox_layers_7_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[91] + gpt_neox_layers_7_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[92] + gpt_neox_layers_7_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[93] + gpt_neox_layers_7_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[94] + gpt_neox_layers_7_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[95] + gpt_neox_layers_7_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[96] + gpt_neox_layers_8_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[97] + gpt_neox_layers_8_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[98] + gpt_neox_layers_8_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[99] + gpt_neox_layers_8_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[100] + gpt_neox_layers_8_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[101] + gpt_neox_layers_8_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[102] + gpt_neox_layers_8_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[103] + gpt_neox_layers_8_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[104] + gpt_neox_layers_8_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[105] + gpt_neox_layers_8_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[106] + gpt_neox_layers_8_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[107] + gpt_neox_layers_8_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[108] + gpt_neox_layers_9_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[109] + gpt_neox_layers_9_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[110] + gpt_neox_layers_9_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[111] + gpt_neox_layers_9_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[112] + gpt_neox_layers_9_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[113] + gpt_neox_layers_9_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[114] + gpt_neox_layers_9_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[115] + gpt_neox_layers_9_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[116] + gpt_neox_layers_9_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[117] + gpt_neox_layers_9_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[118] + gpt_neox_layers_9_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[119] + gpt_neox_layers_9_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[120] + gpt_neox_layers_10_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[121] + gpt_neox_layers_10_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[122] + gpt_neox_layers_10_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[123] + gpt_neox_layers_10_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[124] + gpt_neox_layers_10_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[125] + gpt_neox_layers_10_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[126] + gpt_neox_layers_10_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[127] + gpt_neox_layers_10_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[128] + gpt_neox_layers_10_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[129] + gpt_neox_layers_10_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[130] + gpt_neox_layers_10_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[131] + gpt_neox_layers_10_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[132] + gpt_neox_layers_11_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[133] + gpt_neox_layers_11_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[134] + gpt_neox_layers_11_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[135] + gpt_neox_layers_11_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[136] + gpt_neox_layers_11_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[137] + gpt_neox_layers_11_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[138] + gpt_neox_layers_11_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[139] + gpt_neox_layers_11_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[140] + gpt_neox_layers_11_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[141] + gpt_neox_layers_11_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[142] + gpt_neox_layers_11_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[143] + gpt_neox_layers_11_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[144] + gpt_neox_layers_12_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[145] + gpt_neox_layers_12_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[146] + gpt_neox_layers_12_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[147] + gpt_neox_layers_12_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[148] + gpt_neox_layers_12_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[149] + gpt_neox_layers_12_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[150] + gpt_neox_layers_12_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[151] + gpt_neox_layers_12_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[152] + gpt_neox_layers_12_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[153] + gpt_neox_layers_12_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[154] + gpt_neox_layers_12_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[155] + gpt_neox_layers_12_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[156] + gpt_neox_layers_13_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[157] + gpt_neox_layers_13_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[158] + gpt_neox_layers_13_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[159] + gpt_neox_layers_13_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[160] + gpt_neox_layers_13_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[161] + gpt_neox_layers_13_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[162] + gpt_neox_layers_13_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[163] + gpt_neox_layers_13_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[164] + gpt_neox_layers_13_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[165] + gpt_neox_layers_13_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[166] + gpt_neox_layers_13_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[167] + gpt_neox_layers_13_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[168] + gpt_neox_layers_14_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[169] + gpt_neox_layers_14_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[170] + gpt_neox_layers_14_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[171] + gpt_neox_layers_14_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[172] + gpt_neox_layers_14_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[173] + gpt_neox_layers_14_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[174] + gpt_neox_layers_14_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[175] + gpt_neox_layers_14_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[176] + gpt_neox_layers_14_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[177] + gpt_neox_layers_14_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[178] + gpt_neox_layers_14_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[179] + gpt_neox_layers_14_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[180] + gpt_neox_layers_15_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[181] + gpt_neox_layers_15_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[182] + gpt_neox_layers_15_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[183] + gpt_neox_layers_15_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[184] + gpt_neox_layers_15_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[185] + gpt_neox_layers_15_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[186] + gpt_neox_layers_15_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[187] + gpt_neox_layers_15_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[188] + gpt_neox_layers_15_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[189] + gpt_neox_layers_15_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[190] + gpt_neox_layers_15_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[191] + gpt_neox_layers_15_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[192] + gpt_neox_layers_16_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[193] + gpt_neox_layers_16_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[194] + gpt_neox_layers_16_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[195] + gpt_neox_layers_16_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[196] + gpt_neox_layers_16_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[197] + gpt_neox_layers_16_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[198] + gpt_neox_layers_16_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[199] + gpt_neox_layers_16_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[200] + gpt_neox_layers_16_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[201] + gpt_neox_layers_16_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[202] + gpt_neox_layers_16_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[203] + gpt_neox_layers_16_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[204] + gpt_neox_layers_17_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[205] + gpt_neox_layers_17_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[206] + gpt_neox_layers_17_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[207] + gpt_neox_layers_17_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[208] + gpt_neox_layers_17_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[209] + gpt_neox_layers_17_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[210] + gpt_neox_layers_17_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[211] + gpt_neox_layers_17_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[212] + gpt_neox_layers_17_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[213] + gpt_neox_layers_17_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[214] + gpt_neox_layers_17_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[215] + gpt_neox_layers_17_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[216] + gpt_neox_layers_18_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[217] + gpt_neox_layers_18_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[218] + gpt_neox_layers_18_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[219] + gpt_neox_layers_18_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[220] + gpt_neox_layers_18_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[221] + gpt_neox_layers_18_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[222] + gpt_neox_layers_18_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[223] + gpt_neox_layers_18_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[224] + gpt_neox_layers_18_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[225] + gpt_neox_layers_18_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[226] + gpt_neox_layers_18_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[227] + gpt_neox_layers_18_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[228] + gpt_neox_layers_19_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[229] + gpt_neox_layers_19_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[230] + gpt_neox_layers_19_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[231] + gpt_neox_layers_19_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[232] + gpt_neox_layers_19_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[233] + gpt_neox_layers_19_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[234] + gpt_neox_layers_19_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[235] + gpt_neox_layers_19_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[236] + gpt_neox_layers_19_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[237] + gpt_neox_layers_19_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[238] + gpt_neox_layers_19_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[239] + gpt_neox_layers_19_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[240] + gpt_neox_layers_20_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[241] + gpt_neox_layers_20_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[242] + gpt_neox_layers_20_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[243] + gpt_neox_layers_20_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[244] + gpt_neox_layers_20_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[245] + gpt_neox_layers_20_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[246] + gpt_neox_layers_20_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[247] + gpt_neox_layers_20_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[248] + gpt_neox_layers_20_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[249] + gpt_neox_layers_20_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[250] + gpt_neox_layers_20_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[251] + gpt_neox_layers_20_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[252] + gpt_neox_layers_21_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[253] + gpt_neox_layers_21_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[254] + gpt_neox_layers_21_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[255] + gpt_neox_layers_21_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[256] + gpt_neox_layers_21_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[257] + gpt_neox_layers_21_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[258] + gpt_neox_layers_21_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[259] + gpt_neox_layers_21_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[260] + gpt_neox_layers_21_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[261] + gpt_neox_layers_21_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[262] + gpt_neox_layers_21_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[263] + gpt_neox_layers_21_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[264] + gpt_neox_layers_22_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[265] + gpt_neox_layers_22_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[266] + gpt_neox_layers_22_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[267] + gpt_neox_layers_22_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[268] + gpt_neox_layers_22_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[269] + gpt_neox_layers_22_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[270] + gpt_neox_layers_22_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[271] + gpt_neox_layers_22_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[272] + gpt_neox_layers_22_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[273] + gpt_neox_layers_22_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[274] + gpt_neox_layers_22_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[275] + gpt_neox_layers_22_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[276] + gpt_neox_layers_23_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[277] + gpt_neox_layers_23_input_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[278] + gpt_neox_layers_23_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[279] + gpt_neox_layers_23_post_attention_layernorm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[280] + gpt_neox_layers_23_attention_query_key_value_weight1: R.Tensor((6144, 2048), dtype="float16") = packed_params[281] + gpt_neox_layers_23_attention_query_key_value_bias1: R.Tensor((6144,), dtype="float16") = packed_params[282] + gpt_neox_layers_23_attention_dense_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[283] + gpt_neox_layers_23_attention_dense_bias1: R.Tensor((2048,), dtype="float16") = packed_params[284] + gpt_neox_layers_23_mlp_dense_h_to_4h_weight1: R.Tensor((8192, 2048), dtype="float16") = packed_params[285] + gpt_neox_layers_23_mlp_dense_h_to_4h_bias1: R.Tensor((8192,), dtype="float32") = packed_params[286] + gpt_neox_layers_23_mlp_dense_4h_to_h_weight1: R.Tensor((2048, 8192), dtype="float16") = packed_params[287] + gpt_neox_layers_23_mlp_dense_4h_to_h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[288] + gpt_neox_final_layer_norm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[289] + gpt_neox_final_layer_norm_bias1: R.Tensor((2048,), dtype="float16") = packed_params[290] + embed_out_weight1: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[291] + layer_norm: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_input_layernorm_weight1, gpt_neox_layers_0_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_query_key_value_weight1, axes=None) + matmul: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm, permute_dims, out_dtype="void") + add: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul, gpt_neox_layers_0_attention_query_key_value_bias1) + reshape: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add, R.shape([1, seq_len, 48, 128])) + reshape1: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape, R.shape([seq_len, 48, 128])) + lv = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape2: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv, R.shape([1, seq_len, 16, 128])) + reshape3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape2, R.shape([1, seq_len, 2048])) + permute_dims1: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_attention_dense_weight1, axes=None) + matmul1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape3, permute_dims1, out_dtype="void") + add1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul1, gpt_neox_layers_0_attention_dense_bias1) + layer_norm1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight1, gpt_neox_layers_0_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims2: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_h_to_4h_weight1, axes=None) + matmul2: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm1, permute_dims2, out_dtype="float32") + add2: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul2, gpt_neox_layers_0_mlp_dense_h_to_4h_bias1) + gelu: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add2) + astype: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu, dtype="float16") + permute_dims3: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_0_mlp_dense_4h_to_h_weight1, axes=None) + matmul3: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype, permute_dims3, out_dtype="float32") + add3: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul3, gpt_neox_layers_0_mlp_dense_4h_to_h_bias1) + astype1: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add3, dtype="float16") + add4: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype1, add1) + add5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add4, input_embed) + layer_norm2: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add5, gpt_neox_layers_1_input_layernorm_weight1, gpt_neox_layers_1_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims4: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_query_key_value_weight1, axes=None) + matmul4: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm2, permute_dims4, out_dtype="void") + add6: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul4, gpt_neox_layers_1_attention_query_key_value_bias1) + reshape4: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add6, R.shape([1, seq_len, 48, 128])) + reshape5: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape4, R.shape([seq_len, 48, 128])) + lv1 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape6: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv1, R.shape([1, seq_len, 16, 128])) + reshape7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape6, R.shape([1, seq_len, 2048])) + permute_dims5: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_attention_dense_weight1, axes=None) + matmul5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape7, permute_dims5, out_dtype="void") + add7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul5, gpt_neox_layers_1_attention_dense_bias1) + layer_norm3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add5, gpt_neox_layers_1_post_attention_layernorm_weight1, gpt_neox_layers_1_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims6: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_h_to_4h_weight1, axes=None) + matmul6: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm3, permute_dims6, out_dtype="float32") + add8: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul6, gpt_neox_layers_1_mlp_dense_h_to_4h_bias1) + gelu1: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add8) + astype2: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu1, dtype="float16") + permute_dims7: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_1_mlp_dense_4h_to_h_weight1, axes=None) + matmul7: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype2, permute_dims7, out_dtype="float32") + add9: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul7, gpt_neox_layers_1_mlp_dense_4h_to_h_bias1) + astype3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add9, dtype="float16") + add10: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype3, add7) + add11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add10, add5) + layer_norm4: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add11, gpt_neox_layers_2_input_layernorm_weight1, gpt_neox_layers_2_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims8: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_query_key_value_weight1, axes=None) + matmul8: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm4, permute_dims8, out_dtype="void") + add12: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul8, gpt_neox_layers_2_attention_query_key_value_bias1) + reshape8: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add12, R.shape([1, seq_len, 48, 128])) + reshape9: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape8, R.shape([seq_len, 48, 128])) + lv2 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape10: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv2, R.shape([1, seq_len, 16, 128])) + reshape11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape10, R.shape([1, seq_len, 2048])) + permute_dims9: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_attention_dense_weight1, axes=None) + matmul9: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape11, permute_dims9, out_dtype="void") + add13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul9, gpt_neox_layers_2_attention_dense_bias1) + layer_norm5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add11, gpt_neox_layers_2_post_attention_layernorm_weight1, gpt_neox_layers_2_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims10: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_h_to_4h_weight1, axes=None) + matmul10: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm5, permute_dims10, out_dtype="float32") + add14: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul10, gpt_neox_layers_2_mlp_dense_h_to_4h_bias1) + gelu2: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add14) + astype4: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu2, dtype="float16") + permute_dims11: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_2_mlp_dense_4h_to_h_weight1, axes=None) + matmul11: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype4, permute_dims11, out_dtype="float32") + add15: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul11, gpt_neox_layers_2_mlp_dense_4h_to_h_bias1) + astype5: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add15, dtype="float16") + add16: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype5, add13) + add17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add16, add11) + layer_norm6: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add17, gpt_neox_layers_3_input_layernorm_weight1, gpt_neox_layers_3_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims12: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_query_key_value_weight1, axes=None) + matmul12: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm6, permute_dims12, out_dtype="void") + add18: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul12, gpt_neox_layers_3_attention_query_key_value_bias1) + reshape12: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add18, R.shape([1, seq_len, 48, 128])) + reshape13: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape12, R.shape([seq_len, 48, 128])) + lv3 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape14: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv3, R.shape([1, seq_len, 16, 128])) + reshape15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape14, R.shape([1, seq_len, 2048])) + permute_dims13: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_attention_dense_weight1, axes=None) + matmul13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape15, permute_dims13, out_dtype="void") + add19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul13, gpt_neox_layers_3_attention_dense_bias1) + layer_norm7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add17, gpt_neox_layers_3_post_attention_layernorm_weight1, gpt_neox_layers_3_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims14: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_h_to_4h_weight1, axes=None) + matmul14: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm7, permute_dims14, out_dtype="float32") + add20: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul14, gpt_neox_layers_3_mlp_dense_h_to_4h_bias1) + gelu3: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add20) + astype6: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu3, dtype="float16") + permute_dims15: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_3_mlp_dense_4h_to_h_weight1, axes=None) + matmul15: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype6, permute_dims15, out_dtype="float32") + add21: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul15, gpt_neox_layers_3_mlp_dense_4h_to_h_bias1) + astype7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add21, dtype="float16") + add22: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype7, add19) + add23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add22, add17) + layer_norm8: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add23, gpt_neox_layers_4_input_layernorm_weight1, gpt_neox_layers_4_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims16: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_query_key_value_weight1, axes=None) + matmul16: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm8, permute_dims16, out_dtype="void") + add24: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul16, gpt_neox_layers_4_attention_query_key_value_bias1) + reshape16: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add24, R.shape([1, seq_len, 48, 128])) + reshape17: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape16, R.shape([seq_len, 48, 128])) + lv4 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape18: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv4, R.shape([1, seq_len, 16, 128])) + reshape19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape18, R.shape([1, seq_len, 2048])) + permute_dims17: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_attention_dense_weight1, axes=None) + matmul17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape19, permute_dims17, out_dtype="void") + add25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul17, gpt_neox_layers_4_attention_dense_bias1) + layer_norm9: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add23, gpt_neox_layers_4_post_attention_layernorm_weight1, gpt_neox_layers_4_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims18: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_h_to_4h_weight1, axes=None) + matmul18: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm9, permute_dims18, out_dtype="float32") + add26: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul18, gpt_neox_layers_4_mlp_dense_h_to_4h_bias1) + gelu4: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add26) + astype8: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu4, dtype="float16") + permute_dims19: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_4_mlp_dense_4h_to_h_weight1, axes=None) + matmul19: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype8, permute_dims19, out_dtype="float32") + add27: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul19, gpt_neox_layers_4_mlp_dense_4h_to_h_bias1) + astype9: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add27, dtype="float16") + add28: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype9, add25) + add29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add28, add23) + layer_norm10: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add29, gpt_neox_layers_5_input_layernorm_weight1, gpt_neox_layers_5_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims20: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_query_key_value_weight1, axes=None) + matmul20: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm10, permute_dims20, out_dtype="void") + add30: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul20, gpt_neox_layers_5_attention_query_key_value_bias1) + reshape20: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add30, R.shape([1, seq_len, 48, 128])) + reshape21: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape20, R.shape([seq_len, 48, 128])) + lv5 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape22: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv5, R.shape([1, seq_len, 16, 128])) + reshape23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape22, R.shape([1, seq_len, 2048])) + permute_dims21: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_attention_dense_weight1, axes=None) + matmul21: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape23, permute_dims21, out_dtype="void") + add31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul21, gpt_neox_layers_5_attention_dense_bias1) + layer_norm11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add29, gpt_neox_layers_5_post_attention_layernorm_weight1, gpt_neox_layers_5_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims22: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_h_to_4h_weight1, axes=None) + matmul22: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm11, permute_dims22, out_dtype="float32") + add32: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul22, gpt_neox_layers_5_mlp_dense_h_to_4h_bias1) + gelu5: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add32) + astype10: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu5, dtype="float16") + permute_dims23: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_5_mlp_dense_4h_to_h_weight1, axes=None) + matmul23: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype10, permute_dims23, out_dtype="float32") + add33: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul23, gpt_neox_layers_5_mlp_dense_4h_to_h_bias1) + astype11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add33, dtype="float16") + add34: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype11, add31) + add35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add34, add29) + layer_norm12: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add35, gpt_neox_layers_6_input_layernorm_weight1, gpt_neox_layers_6_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims24: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_query_key_value_weight1, axes=None) + matmul24: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm12, permute_dims24, out_dtype="void") + add36: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul24, gpt_neox_layers_6_attention_query_key_value_bias1) + reshape24: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add36, R.shape([1, seq_len, 48, 128])) + reshape25: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape24, R.shape([seq_len, 48, 128])) + lv6 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape25), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape26: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv6, R.shape([1, seq_len, 16, 128])) + reshape27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape26, R.shape([1, seq_len, 2048])) + permute_dims25: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_attention_dense_weight1, axes=None) + matmul25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape27, permute_dims25, out_dtype="void") + add37: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul25, gpt_neox_layers_6_attention_dense_bias1) + layer_norm13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add35, gpt_neox_layers_6_post_attention_layernorm_weight1, gpt_neox_layers_6_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims26: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_h_to_4h_weight1, axes=None) + matmul26: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm13, permute_dims26, out_dtype="float32") + add38: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul26, gpt_neox_layers_6_mlp_dense_h_to_4h_bias1) + gelu6: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add38) + astype12: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu6, dtype="float16") + permute_dims27: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_6_mlp_dense_4h_to_h_weight1, axes=None) + matmul27: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype12, permute_dims27, out_dtype="float32") + add39: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul27, gpt_neox_layers_6_mlp_dense_4h_to_h_bias1) + astype13: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add39, dtype="float16") + add40: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype13, add37) + add41: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add40, add35) + layer_norm14: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add41, gpt_neox_layers_7_input_layernorm_weight1, gpt_neox_layers_7_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims28: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_query_key_value_weight1, axes=None) + matmul28: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm14, permute_dims28, out_dtype="void") + add42: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul28, gpt_neox_layers_7_attention_query_key_value_bias1) + reshape28: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add42, R.shape([1, seq_len, 48, 128])) + reshape29: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape28, R.shape([seq_len, 48, 128])) + lv7 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape29), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape30: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv7, R.shape([1, seq_len, 16, 128])) + reshape31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape30, R.shape([1, seq_len, 2048])) + permute_dims29: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_attention_dense_weight1, axes=None) + matmul29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape31, permute_dims29, out_dtype="void") + add43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul29, gpt_neox_layers_7_attention_dense_bias1) + layer_norm15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add41, gpt_neox_layers_7_post_attention_layernorm_weight1, gpt_neox_layers_7_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims30: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_h_to_4h_weight1, axes=None) + matmul30: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm15, permute_dims30, out_dtype="float32") + add44: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul30, gpt_neox_layers_7_mlp_dense_h_to_4h_bias1) + gelu7: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add44) + astype14: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu7, dtype="float16") + permute_dims31: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_7_mlp_dense_4h_to_h_weight1, axes=None) + matmul31: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype14, permute_dims31, out_dtype="float32") + add45: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul31, gpt_neox_layers_7_mlp_dense_4h_to_h_bias1) + astype15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add45, dtype="float16") + add46: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype15, add43) + add47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add46, add41) + layer_norm16: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add47, gpt_neox_layers_8_input_layernorm_weight1, gpt_neox_layers_8_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims32: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_query_key_value_weight1, axes=None) + matmul32: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm16, permute_dims32, out_dtype="void") + add48: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul32, gpt_neox_layers_8_attention_query_key_value_bias1) + reshape32: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add48, R.shape([1, seq_len, 48, 128])) + reshape33: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape32, R.shape([seq_len, 48, 128])) + lv8 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape33), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape34: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv8, R.shape([1, seq_len, 16, 128])) + reshape35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape34, R.shape([1, seq_len, 2048])) + permute_dims33: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_attention_dense_weight1, axes=None) + matmul33: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape35, permute_dims33, out_dtype="void") + add49: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul33, gpt_neox_layers_8_attention_dense_bias1) + layer_norm17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add47, gpt_neox_layers_8_post_attention_layernorm_weight1, gpt_neox_layers_8_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims34: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_h_to_4h_weight1, axes=None) + matmul34: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm17, permute_dims34, out_dtype="float32") + add50: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul34, gpt_neox_layers_8_mlp_dense_h_to_4h_bias1) + gelu8: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add50) + astype16: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu8, dtype="float16") + permute_dims35: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_8_mlp_dense_4h_to_h_weight1, axes=None) + matmul35: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype16, permute_dims35, out_dtype="float32") + add51: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul35, gpt_neox_layers_8_mlp_dense_4h_to_h_bias1) + astype17: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add51, dtype="float16") + add52: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype17, add49) + add53: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add52, add47) + layer_norm18: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add53, gpt_neox_layers_9_input_layernorm_weight1, gpt_neox_layers_9_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims36: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_query_key_value_weight1, axes=None) + matmul36: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm18, permute_dims36, out_dtype="void") + add54: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul36, gpt_neox_layers_9_attention_query_key_value_bias1) + reshape36: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add54, R.shape([1, seq_len, 48, 128])) + reshape37: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape36, R.shape([seq_len, 48, 128])) + lv9 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape37), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape38: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv9, R.shape([1, seq_len, 16, 128])) + reshape39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape38, R.shape([1, seq_len, 2048])) + permute_dims37: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_attention_dense_weight1, axes=None) + matmul37: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape39, permute_dims37, out_dtype="void") + add55: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul37, gpt_neox_layers_9_attention_dense_bias1) + layer_norm19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add53, gpt_neox_layers_9_post_attention_layernorm_weight1, gpt_neox_layers_9_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims38: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_h_to_4h_weight1, axes=None) + matmul38: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm19, permute_dims38, out_dtype="float32") + add56: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul38, gpt_neox_layers_9_mlp_dense_h_to_4h_bias1) + gelu9: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add56) + astype18: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu9, dtype="float16") + permute_dims39: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_9_mlp_dense_4h_to_h_weight1, axes=None) + matmul39: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype18, permute_dims39, out_dtype="float32") + add57: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul39, gpt_neox_layers_9_mlp_dense_4h_to_h_bias1) + astype19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add57, dtype="float16") + add58: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype19, add55) + add59: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add58, add53) + layer_norm20: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add59, gpt_neox_layers_10_input_layernorm_weight1, gpt_neox_layers_10_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims40: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_query_key_value_weight1, axes=None) + matmul40: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm20, permute_dims40, out_dtype="void") + add60: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul40, gpt_neox_layers_10_attention_query_key_value_bias1) + reshape40: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add60, R.shape([1, seq_len, 48, 128])) + reshape41: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape40, R.shape([seq_len, 48, 128])) + lv10 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape41), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape42: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv10, R.shape([1, seq_len, 16, 128])) + reshape43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape42, R.shape([1, seq_len, 2048])) + permute_dims41: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_attention_dense_weight1, axes=None) + matmul41: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape43, permute_dims41, out_dtype="void") + add61: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul41, gpt_neox_layers_10_attention_dense_bias1) + layer_norm21: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add59, gpt_neox_layers_10_post_attention_layernorm_weight1, gpt_neox_layers_10_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims42: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_h_to_4h_weight1, axes=None) + matmul42: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm21, permute_dims42, out_dtype="float32") + add62: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul42, gpt_neox_layers_10_mlp_dense_h_to_4h_bias1) + gelu10: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add62) + astype20: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu10, dtype="float16") + permute_dims43: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_10_mlp_dense_4h_to_h_weight1, axes=None) + matmul43: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype20, permute_dims43, out_dtype="float32") + add63: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul43, gpt_neox_layers_10_mlp_dense_4h_to_h_bias1) + astype21: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add63, dtype="float16") + add64: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype21, add61) + add65: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add64, add59) + layer_norm22: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add65, gpt_neox_layers_11_input_layernorm_weight1, gpt_neox_layers_11_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims44: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_query_key_value_weight1, axes=None) + matmul44: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm22, permute_dims44, out_dtype="void") + add66: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul44, gpt_neox_layers_11_attention_query_key_value_bias1) + reshape44: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add66, R.shape([1, seq_len, 48, 128])) + reshape45: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape44, R.shape([seq_len, 48, 128])) + lv11 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape45), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape46: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv11, R.shape([1, seq_len, 16, 128])) + reshape47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape46, R.shape([1, seq_len, 2048])) + permute_dims45: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_attention_dense_weight1, axes=None) + matmul45: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape47, permute_dims45, out_dtype="void") + add67: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul45, gpt_neox_layers_11_attention_dense_bias1) + layer_norm23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add65, gpt_neox_layers_11_post_attention_layernorm_weight1, gpt_neox_layers_11_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims46: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_h_to_4h_weight1, axes=None) + matmul46: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm23, permute_dims46, out_dtype="float32") + add68: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul46, gpt_neox_layers_11_mlp_dense_h_to_4h_bias1) + gelu11: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add68) + astype22: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu11, dtype="float16") + permute_dims47: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_11_mlp_dense_4h_to_h_weight1, axes=None) + matmul47: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype22, permute_dims47, out_dtype="float32") + add69: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul47, gpt_neox_layers_11_mlp_dense_4h_to_h_bias1) + astype23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add69, dtype="float16") + add70: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype23, add67) + add71: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add70, add65) + layer_norm24: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add71, gpt_neox_layers_12_input_layernorm_weight1, gpt_neox_layers_12_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims48: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_query_key_value_weight1, axes=None) + matmul48: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm24, permute_dims48, out_dtype="void") + add72: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul48, gpt_neox_layers_12_attention_query_key_value_bias1) + reshape48: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add72, R.shape([1, seq_len, 48, 128])) + reshape49: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape48, R.shape([seq_len, 48, 128])) + lv12 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape49), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape50: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv12, R.shape([1, seq_len, 16, 128])) + reshape51: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape50, R.shape([1, seq_len, 2048])) + permute_dims49: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_attention_dense_weight1, axes=None) + matmul49: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape51, permute_dims49, out_dtype="void") + add73: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul49, gpt_neox_layers_12_attention_dense_bias1) + layer_norm25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add71, gpt_neox_layers_12_post_attention_layernorm_weight1, gpt_neox_layers_12_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims50: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_h_to_4h_weight1, axes=None) + matmul50: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm25, permute_dims50, out_dtype="float32") + add74: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul50, gpt_neox_layers_12_mlp_dense_h_to_4h_bias1) + gelu12: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add74) + astype24: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu12, dtype="float16") + permute_dims51: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_12_mlp_dense_4h_to_h_weight1, axes=None) + matmul51: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype24, permute_dims51, out_dtype="float32") + add75: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul51, gpt_neox_layers_12_mlp_dense_4h_to_h_bias1) + astype25: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add75, dtype="float16") + add76: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype25, add73) + add77: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add76, add71) + layer_norm26: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add77, gpt_neox_layers_13_input_layernorm_weight1, gpt_neox_layers_13_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims52: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_query_key_value_weight1, axes=None) + matmul52: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm26, permute_dims52, out_dtype="void") + add78: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul52, gpt_neox_layers_13_attention_query_key_value_bias1) + reshape52: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add78, R.shape([1, seq_len, 48, 128])) + reshape53: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape52, R.shape([seq_len, 48, 128])) + lv13 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape53), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape54: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv13, R.shape([1, seq_len, 16, 128])) + reshape55: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape54, R.shape([1, seq_len, 2048])) + permute_dims53: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_attention_dense_weight1, axes=None) + matmul53: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape55, permute_dims53, out_dtype="void") + add79: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul53, gpt_neox_layers_13_attention_dense_bias1) + layer_norm27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add77, gpt_neox_layers_13_post_attention_layernorm_weight1, gpt_neox_layers_13_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims54: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_h_to_4h_weight1, axes=None) + matmul54: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm27, permute_dims54, out_dtype="float32") + add80: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul54, gpt_neox_layers_13_mlp_dense_h_to_4h_bias1) + gelu13: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add80) + astype26: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu13, dtype="float16") + permute_dims55: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_13_mlp_dense_4h_to_h_weight1, axes=None) + matmul55: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype26, permute_dims55, out_dtype="float32") + add81: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul55, gpt_neox_layers_13_mlp_dense_4h_to_h_bias1) + astype27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add81, dtype="float16") + add82: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype27, add79) + add83: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add82, add77) + layer_norm28: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add83, gpt_neox_layers_14_input_layernorm_weight1, gpt_neox_layers_14_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims56: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_query_key_value_weight1, axes=None) + matmul56: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm28, permute_dims56, out_dtype="void") + add84: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul56, gpt_neox_layers_14_attention_query_key_value_bias1) + reshape56: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add84, R.shape([1, seq_len, 48, 128])) + reshape57: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape56, R.shape([seq_len, 48, 128])) + lv14 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape57), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape58: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv14, R.shape([1, seq_len, 16, 128])) + reshape59: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape58, R.shape([1, seq_len, 2048])) + permute_dims57: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_attention_dense_weight1, axes=None) + matmul57: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape59, permute_dims57, out_dtype="void") + add85: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul57, gpt_neox_layers_14_attention_dense_bias1) + layer_norm29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add83, gpt_neox_layers_14_post_attention_layernorm_weight1, gpt_neox_layers_14_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims58: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_h_to_4h_weight1, axes=None) + matmul58: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm29, permute_dims58, out_dtype="float32") + add86: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul58, gpt_neox_layers_14_mlp_dense_h_to_4h_bias1) + gelu14: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add86) + astype28: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu14, dtype="float16") + permute_dims59: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_14_mlp_dense_4h_to_h_weight1, axes=None) + matmul59: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype28, permute_dims59, out_dtype="float32") + add87: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul59, gpt_neox_layers_14_mlp_dense_4h_to_h_bias1) + astype29: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add87, dtype="float16") + add88: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype29, add85) + add89: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add88, add83) + layer_norm30: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add89, gpt_neox_layers_15_input_layernorm_weight1, gpt_neox_layers_15_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims60: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_query_key_value_weight1, axes=None) + matmul60: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm30, permute_dims60, out_dtype="void") + add90: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul60, gpt_neox_layers_15_attention_query_key_value_bias1) + reshape60: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add90, R.shape([1, seq_len, 48, 128])) + reshape61: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape60, R.shape([seq_len, 48, 128])) + lv15 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape61), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape62: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv15, R.shape([1, seq_len, 16, 128])) + reshape63: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape62, R.shape([1, seq_len, 2048])) + permute_dims61: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_attention_dense_weight1, axes=None) + matmul61: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape63, permute_dims61, out_dtype="void") + add91: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul61, gpt_neox_layers_15_attention_dense_bias1) + layer_norm31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add89, gpt_neox_layers_15_post_attention_layernorm_weight1, gpt_neox_layers_15_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims62: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_h_to_4h_weight1, axes=None) + matmul62: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm31, permute_dims62, out_dtype="float32") + add92: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul62, gpt_neox_layers_15_mlp_dense_h_to_4h_bias1) + gelu15: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add92) + astype30: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu15, dtype="float16") + permute_dims63: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_15_mlp_dense_4h_to_h_weight1, axes=None) + matmul63: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype30, permute_dims63, out_dtype="float32") + add93: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul63, gpt_neox_layers_15_mlp_dense_4h_to_h_bias1) + astype31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add93, dtype="float16") + add94: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype31, add91) + add95: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add94, add89) + layer_norm32: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add95, gpt_neox_layers_16_input_layernorm_weight1, gpt_neox_layers_16_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims64: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_query_key_value_weight1, axes=None) + matmul64: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm32, permute_dims64, out_dtype="void") + add96: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul64, gpt_neox_layers_16_attention_query_key_value_bias1) + reshape64: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add96, R.shape([1, seq_len, 48, 128])) + reshape65: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape64, R.shape([seq_len, 48, 128])) + lv16 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape65), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape66: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv16, R.shape([1, seq_len, 16, 128])) + reshape67: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape66, R.shape([1, seq_len, 2048])) + permute_dims65: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_attention_dense_weight1, axes=None) + matmul65: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape67, permute_dims65, out_dtype="void") + add97: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul65, gpt_neox_layers_16_attention_dense_bias1) + layer_norm33: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add95, gpt_neox_layers_16_post_attention_layernorm_weight1, gpt_neox_layers_16_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims66: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_h_to_4h_weight1, axes=None) + matmul66: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm33, permute_dims66, out_dtype="float32") + add98: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul66, gpt_neox_layers_16_mlp_dense_h_to_4h_bias1) + gelu16: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add98) + astype32: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu16, dtype="float16") + permute_dims67: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_16_mlp_dense_4h_to_h_weight1, axes=None) + matmul67: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype32, permute_dims67, out_dtype="float32") + add99: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul67, gpt_neox_layers_16_mlp_dense_4h_to_h_bias1) + astype33: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add99, dtype="float16") + add100: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype33, add97) + add101: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add100, add95) + layer_norm34: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add101, gpt_neox_layers_17_input_layernorm_weight1, gpt_neox_layers_17_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims68: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_query_key_value_weight1, axes=None) + matmul68: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm34, permute_dims68, out_dtype="void") + add102: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul68, gpt_neox_layers_17_attention_query_key_value_bias1) + reshape68: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add102, R.shape([1, seq_len, 48, 128])) + reshape69: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape68, R.shape([seq_len, 48, 128])) + lv17 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape69), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape70: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv17, R.shape([1, seq_len, 16, 128])) + reshape71: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape70, R.shape([1, seq_len, 2048])) + permute_dims69: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_attention_dense_weight1, axes=None) + matmul69: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape71, permute_dims69, out_dtype="void") + add103: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul69, gpt_neox_layers_17_attention_dense_bias1) + layer_norm35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add101, gpt_neox_layers_17_post_attention_layernorm_weight1, gpt_neox_layers_17_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims70: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_h_to_4h_weight1, axes=None) + matmul70: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm35, permute_dims70, out_dtype="float32") + add104: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul70, gpt_neox_layers_17_mlp_dense_h_to_4h_bias1) + gelu17: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add104) + astype34: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu17, dtype="float16") + permute_dims71: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_17_mlp_dense_4h_to_h_weight1, axes=None) + matmul71: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype34, permute_dims71, out_dtype="float32") + add105: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul71, gpt_neox_layers_17_mlp_dense_4h_to_h_bias1) + astype35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add105, dtype="float16") + add106: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype35, add103) + add107: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add106, add101) + layer_norm36: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add107, gpt_neox_layers_18_input_layernorm_weight1, gpt_neox_layers_18_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims72: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_query_key_value_weight1, axes=None) + matmul72: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm36, permute_dims72, out_dtype="void") + add108: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul72, gpt_neox_layers_18_attention_query_key_value_bias1) + reshape72: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add108, R.shape([1, seq_len, 48, 128])) + reshape73: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape72, R.shape([seq_len, 48, 128])) + lv18 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(18), R.prim_value(T.float32(1.0)), reshape73), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape74: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv18, R.shape([1, seq_len, 16, 128])) + reshape75: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape74, R.shape([1, seq_len, 2048])) + permute_dims73: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_attention_dense_weight1, axes=None) + matmul73: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape75, permute_dims73, out_dtype="void") + add109: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul73, gpt_neox_layers_18_attention_dense_bias1) + layer_norm37: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add107, gpt_neox_layers_18_post_attention_layernorm_weight1, gpt_neox_layers_18_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims74: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_h_to_4h_weight1, axes=None) + matmul74: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm37, permute_dims74, out_dtype="float32") + add110: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul74, gpt_neox_layers_18_mlp_dense_h_to_4h_bias1) + gelu18: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add110) + astype36: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu18, dtype="float16") + permute_dims75: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_18_mlp_dense_4h_to_h_weight1, axes=None) + matmul75: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype36, permute_dims75, out_dtype="float32") + add111: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul75, gpt_neox_layers_18_mlp_dense_4h_to_h_bias1) + astype37: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add111, dtype="float16") + add112: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype37, add109) + add113: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add112, add107) + layer_norm38: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add113, gpt_neox_layers_19_input_layernorm_weight1, gpt_neox_layers_19_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims76: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_query_key_value_weight1, axes=None) + matmul76: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm38, permute_dims76, out_dtype="void") + add114: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul76, gpt_neox_layers_19_attention_query_key_value_bias1) + reshape76: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add114, R.shape([1, seq_len, 48, 128])) + reshape77: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape76, R.shape([seq_len, 48, 128])) + lv19 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(19), R.prim_value(T.float32(1.0)), reshape77), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape78: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv19, R.shape([1, seq_len, 16, 128])) + reshape79: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape78, R.shape([1, seq_len, 2048])) + permute_dims77: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_attention_dense_weight1, axes=None) + matmul77: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape79, permute_dims77, out_dtype="void") + add115: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul77, gpt_neox_layers_19_attention_dense_bias1) + layer_norm39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add113, gpt_neox_layers_19_post_attention_layernorm_weight1, gpt_neox_layers_19_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims78: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_h_to_4h_weight1, axes=None) + matmul78: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm39, permute_dims78, out_dtype="float32") + add116: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul78, gpt_neox_layers_19_mlp_dense_h_to_4h_bias1) + gelu19: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add116) + astype38: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu19, dtype="float16") + permute_dims79: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_19_mlp_dense_4h_to_h_weight1, axes=None) + matmul79: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype38, permute_dims79, out_dtype="float32") + add117: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul79, gpt_neox_layers_19_mlp_dense_4h_to_h_bias1) + astype39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add117, dtype="float16") + add118: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype39, add115) + add119: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add118, add113) + layer_norm40: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add119, gpt_neox_layers_20_input_layernorm_weight1, gpt_neox_layers_20_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims80: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_query_key_value_weight1, axes=None) + matmul80: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm40, permute_dims80, out_dtype="void") + add120: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul80, gpt_neox_layers_20_attention_query_key_value_bias1) + reshape80: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add120, R.shape([1, seq_len, 48, 128])) + reshape81: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape80, R.shape([seq_len, 48, 128])) + lv20 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(20), R.prim_value(T.float32(1.0)), reshape81), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape82: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv20, R.shape([1, seq_len, 16, 128])) + reshape83: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape82, R.shape([1, seq_len, 2048])) + permute_dims81: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_attention_dense_weight1, axes=None) + matmul81: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape83, permute_dims81, out_dtype="void") + add121: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul81, gpt_neox_layers_20_attention_dense_bias1) + layer_norm41: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add119, gpt_neox_layers_20_post_attention_layernorm_weight1, gpt_neox_layers_20_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims82: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_h_to_4h_weight1, axes=None) + matmul82: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm41, permute_dims82, out_dtype="float32") + add122: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul82, gpt_neox_layers_20_mlp_dense_h_to_4h_bias1) + gelu20: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add122) + astype40: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu20, dtype="float16") + permute_dims83: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_20_mlp_dense_4h_to_h_weight1, axes=None) + matmul83: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype40, permute_dims83, out_dtype="float32") + add123: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul83, gpt_neox_layers_20_mlp_dense_4h_to_h_bias1) + astype41: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add123, dtype="float16") + add124: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype41, add121) + add125: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add124, add119) + layer_norm42: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add125, gpt_neox_layers_21_input_layernorm_weight1, gpt_neox_layers_21_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims84: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_query_key_value_weight1, axes=None) + matmul84: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm42, permute_dims84, out_dtype="void") + add126: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul84, gpt_neox_layers_21_attention_query_key_value_bias1) + reshape84: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add126, R.shape([1, seq_len, 48, 128])) + reshape85: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape84, R.shape([seq_len, 48, 128])) + lv21 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(21), R.prim_value(T.float32(1.0)), reshape85), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape86: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv21, R.shape([1, seq_len, 16, 128])) + reshape87: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape86, R.shape([1, seq_len, 2048])) + permute_dims85: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_attention_dense_weight1, axes=None) + matmul85: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape87, permute_dims85, out_dtype="void") + add127: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul85, gpt_neox_layers_21_attention_dense_bias1) + layer_norm43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add125, gpt_neox_layers_21_post_attention_layernorm_weight1, gpt_neox_layers_21_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims86: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_h_to_4h_weight1, axes=None) + matmul86: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm43, permute_dims86, out_dtype="float32") + add128: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul86, gpt_neox_layers_21_mlp_dense_h_to_4h_bias1) + gelu21: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add128) + astype42: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu21, dtype="float16") + permute_dims87: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_21_mlp_dense_4h_to_h_weight1, axes=None) + matmul87: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype42, permute_dims87, out_dtype="float32") + add129: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul87, gpt_neox_layers_21_mlp_dense_4h_to_h_bias1) + astype43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add129, dtype="float16") + add130: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype43, add127) + add131: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add130, add125) + layer_norm44: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add131, gpt_neox_layers_22_input_layernorm_weight1, gpt_neox_layers_22_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims88: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_query_key_value_weight1, axes=None) + matmul88: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm44, permute_dims88, out_dtype="void") + add132: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul88, gpt_neox_layers_22_attention_query_key_value_bias1) + reshape88: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add132, R.shape([1, seq_len, 48, 128])) + reshape89: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape88, R.shape([seq_len, 48, 128])) + lv22 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(22), R.prim_value(T.float32(1.0)), reshape89), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape90: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv22, R.shape([1, seq_len, 16, 128])) + reshape91: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape90, R.shape([1, seq_len, 2048])) + permute_dims89: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_attention_dense_weight1, axes=None) + matmul89: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape91, permute_dims89, out_dtype="void") + add133: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul89, gpt_neox_layers_22_attention_dense_bias1) + layer_norm45: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add131, gpt_neox_layers_22_post_attention_layernorm_weight1, gpt_neox_layers_22_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims90: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_h_to_4h_weight1, axes=None) + matmul90: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm45, permute_dims90, out_dtype="float32") + add134: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul90, gpt_neox_layers_22_mlp_dense_h_to_4h_bias1) + gelu22: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add134) + astype44: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu22, dtype="float16") + permute_dims91: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_22_mlp_dense_4h_to_h_weight1, axes=None) + matmul91: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype44, permute_dims91, out_dtype="float32") + add135: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul91, gpt_neox_layers_22_mlp_dense_4h_to_h_bias1) + astype45: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add135, dtype="float16") + add136: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype45, add133) + add137: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add136, add131) + layer_norm46: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add137, gpt_neox_layers_23_input_layernorm_weight1, gpt_neox_layers_23_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims92: R.Tensor((2048, 6144), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_query_key_value_weight1, axes=None) + matmul92: R.Tensor((1, seq_len, 6144), dtype="float16") = R.matmul(layer_norm46, permute_dims92, out_dtype="void") + add138: R.Tensor((1, seq_len, 6144), dtype="float16") = R.add(matmul92, gpt_neox_layers_23_attention_query_key_value_bias1) + reshape92: R.Tensor((1, seq_len, 48, 128), dtype="float16") = R.reshape(add138, R.shape([1, seq_len, 48, 128])) + reshape93: R.Tensor((seq_len, 48, 128), dtype="float16") = R.reshape(reshape92, R.shape([seq_len, 48, 128])) + lv23 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(23), R.prim_value(T.float32(1.0)), reshape93), out_sinfo=R.Tensor((seq_len, 16, 128), dtype="float16")) + reshape94: R.Tensor((1, seq_len, 16, 128), dtype="float16") = R.reshape(lv23, R.shape([1, seq_len, 16, 128])) + reshape95: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape94, R.shape([1, seq_len, 2048])) + permute_dims93: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_attention_dense_weight1, axes=None) + matmul93: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape95, permute_dims93, out_dtype="void") + add139: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(matmul93, gpt_neox_layers_23_attention_dense_bias1) + layer_norm47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add137, gpt_neox_layers_23_post_attention_layernorm_weight1, gpt_neox_layers_23_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + permute_dims94: R.Tensor((2048, 8192), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_h_to_4h_weight1, axes=None) + matmul94: R.Tensor((1, seq_len, 8192), dtype="float32") = R.matmul(layer_norm47, permute_dims94, out_dtype="float32") + add140: R.Tensor((1, seq_len, 8192), dtype="float32") = R.add(matmul94, gpt_neox_layers_23_mlp_dense_h_to_4h_bias1) + gelu23: R.Tensor((1, seq_len, 8192), dtype="float32") = R.nn.gelu(add140) + astype46: R.Tensor((1, seq_len, 8192), dtype="float16") = R.astype(gelu23, dtype="float16") + permute_dims95: R.Tensor((8192, 2048), dtype="float16") = R.permute_dims(gpt_neox_layers_23_mlp_dense_4h_to_h_weight1, axes=None) + matmul95: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(astype46, permute_dims95, out_dtype="float32") + add141: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul95, gpt_neox_layers_23_mlp_dense_4h_to_h_bias1) + astype47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(add141, dtype="float16") + add142: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(astype47, add139) + add143: R.Tensor((1, seq_len, 2048), dtype="float16") = R.add(add142, add137) + layer_norm48: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.layer_norm(add143, gpt_neox_final_layer_norm_weight1, gpt_neox_final_layer_norm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) + lv24 = R.call_tir(cls.index, (layer_norm48,), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) + permute_dims96: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(embed_out_weight1, axes=None) + matmul96: R.Tensor((1, 1, vocab_size), dtype="float16") = R.matmul(lv24, permute_dims96, out_dtype="void") + astype48: R.Tensor((1, 1, vocab_size), dtype="float32") = R.astype(matmul96, dtype="float32") + gv1: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = astype48, paged_kv_cache + R.output(gv1) + return gv1 + + @R.function + def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): + batch_size = T.int64(is_size_var=True) + vocab_size = T.int64(is_size_var=True) + R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) + cls = Module + with R.dataflow(): + lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", logits, R.shape([batch_size, vocab_size]), sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) + lv1 = R.call_tir(cls.chunk_lse, (lv, temperature), out_sinfo=[R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32"), R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32")]) + lv2: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[0] + lv3: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[1] + lv4 = R.call_tir(cls.softmax_with_chunked_sum, (lv, temperature, lv2, lv3), out_sinfo=R.Tensor((batch_size, vocab_size), dtype="float32")) + gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", lv4, R.shape([batch_size, 1, vocab_size]), sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) + R.output(gv) + return gv \ No newline at end of file