# from tvm.script import ir as I # from tvm.script import tir as T # from tvm.script import relax as R @I.ir_module class Module: I.module_attrs({"system_lib_prefix": "gemma_q0f16_"}) @T.prim_func(private=True) def NT_matmul(var_rms_norm111: T.handle, model_layers_0_self_attn_qkv_proj_weight4: T.Buffer((T.int64(2560), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() rms_norm111 = T.match_buffer(var_rms_norm111, (batch_size, T.int64(1), T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(2560)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(2560), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm111[v_i0, v_i1, v_k], model_layers_0_self_attn_qkv_proj_weight4[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + rms_norm111[v_i0, v_i1, v_k] * model_layers_0_self_attn_qkv_proj_weight4[v_i2, v_k] @T.prim_func(private=True) def NT_matmul1(var_reshape219: T.handle, model_layers_0_self_attn_o_proj_weight4: T.Buffer((T.int64(2048), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() reshape219 = T.match_buffer(var_reshape219, (batch_size, T.int64(1), T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(2048)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(2048), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(reshape219[v_i0, v_i1, v_k], model_layers_0_self_attn_o_proj_weight4[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape219[v_i0, v_i1, v_k] * model_layers_0_self_attn_o_proj_weight4[v_i2, v_k] @T.prim_func(private=True) def NT_matmul10(rms_norm37: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16"), model_layers_0_self_attn_qkv_proj_weight2: T.Buffer((T.int64(2560), T.int64(2048)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(2560)), "float16")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(2560), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm37[v_i0, v_i1, v_k], model_layers_0_self_attn_qkv_proj_weight2[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + rms_norm37[v_i0, v_i1, v_k] * model_layers_0_self_attn_qkv_proj_weight2[v_i2, v_k] @T.prim_func(private=True) def NT_matmul11(reshape75: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16"), model_layers_0_self_attn_o_proj_weight2: T.Buffer((T.int64(2048), T.int64(2048)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(2048), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(reshape75[v_i0, v_i1, v_k], model_layers_0_self_attn_o_proj_weight2[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape75[v_i0, v_i1, v_k] * model_layers_0_self_attn_o_proj_weight2[v_i2, v_k] @T.prim_func(private=True) def NT_matmul12(rms_norm38: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16"), model_layers_0_mlp_gate_up_proj_weight2: T.Buffer((T.int64(32768), T.int64(2048)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(32768)), "float16")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(32768), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm38[v_i0, v_i1, v_k], model_layers_0_mlp_gate_up_proj_weight2[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + rms_norm38[v_i0, v_i1, v_k] * model_layers_0_mlp_gate_up_proj_weight2[v_i2, v_k] @T.prim_func(private=True) def NT_matmul13(mul20: T.Buffer((T.int64(1), T.int64(1), T.int64(16384)), "float16"), model_layers_0_mlp_down_proj_weight2: T.Buffer((T.int64(2048), T.int64(16384)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(2048), T.int64(16384)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(mul20[v_i0, v_i1, v_k], model_layers_0_mlp_down_proj_weight2[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + mul20[v_i0, v_i1, v_k] * model_layers_0_mlp_down_proj_weight2[v_i2, v_k] @T.prim_func(private=True) def NT_matmul14(rms_norm73: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16"), var_model_embed_tokens_weight2: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) vocab_size = T.int64() model_embed_tokens_weight2 = T.match_buffer(var_model_embed_tokens_weight2, (vocab_size, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), T.int64(1), vocab_size)) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), vocab_size, T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm73[v_i0, v_i1, v_k], model_embed_tokens_weight2[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", rms_norm73[v_i0, v_i1, v_k]) * T.Cast("float32", model_embed_tokens_weight2[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul2(var_rms_norm112: T.handle, model_layers_0_mlp_gate_up_proj_weight4: T.Buffer((T.int64(32768), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() rms_norm112 = T.match_buffer(var_rms_norm112, (batch_size, T.int64(1), T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(32768)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(32768), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm112[v_i0, v_i1, v_k], model_layers_0_mlp_gate_up_proj_weight4[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + rms_norm112[v_i0, v_i1, v_k] * model_layers_0_mlp_gate_up_proj_weight4[v_i2, v_k] @T.prim_func(private=True) def NT_matmul3(var_mul58: T.handle, model_layers_0_mlp_down_proj_weight4: T.Buffer((T.int64(2048), T.int64(16384)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() mul58 = T.match_buffer(var_mul58, (batch_size, T.int64(1), T.int64(16384)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(2048)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(2048), T.int64(16384)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(mul58[v_i0, v_i1, v_k], model_layers_0_mlp_down_proj_weight4[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + mul58[v_i0, v_i1, v_k] * model_layers_0_mlp_down_proj_weight4[v_i2, v_k] @T.prim_func(private=True) def NT_matmul4(var_rms_norm147: T.handle, var_model_embed_tokens_weight4: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() rms_norm147 = T.match_buffer(var_rms_norm147, (batch_size, T.int64(1), T.int64(2048)), "float16") vocab_size = T.int64() model_embed_tokens_weight4 = T.match_buffer(var_model_embed_tokens_weight4, (vocab_size, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), vocab_size)) # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), vocab_size, T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm147[v_i0, v_i1, v_k], model_embed_tokens_weight4[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", rms_norm147[v_i0, v_i1, v_k]) * T.Cast("float32", model_embed_tokens_weight4[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul5(var_rms_norm74: T.handle, model_layers_0_self_attn_qkv_proj_weight3: T.Buffer((T.int64(2560), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() rms_norm74 = T.match_buffer(var_rms_norm74, (T.int64(1), seq_len, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(2560)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(2560), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm74[v_i0, v_i1, v_k], model_layers_0_self_attn_qkv_proj_weight3[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + rms_norm74[v_i0, v_i1, v_k] * model_layers_0_self_attn_qkv_proj_weight3[v_i2, v_k] @T.prim_func(private=True) def NT_matmul6(var_reshape147: T.handle, model_layers_0_self_attn_o_proj_weight3: T.Buffer((T.int64(2048), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() reshape147 = T.match_buffer(var_reshape147, (T.int64(1), seq_len, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(2048)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(2048), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(reshape147[v_i0, v_i1, v_k], model_layers_0_self_attn_o_proj_weight3[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape147[v_i0, v_i1, v_k] * model_layers_0_self_attn_o_proj_weight3[v_i2, v_k] @T.prim_func(private=True) def NT_matmul7(var_rms_norm75: T.handle, model_layers_0_mlp_gate_up_proj_weight3: T.Buffer((T.int64(32768), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() rms_norm75 = T.match_buffer(var_rms_norm75, (T.int64(1), seq_len, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(32768)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(32768), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(rms_norm75[v_i0, v_i1, v_k], model_layers_0_mlp_gate_up_proj_weight3[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + rms_norm75[v_i0, v_i1, v_k] * model_layers_0_mlp_gate_up_proj_weight3[v_i2, v_k] @T.prim_func(private=True) def NT_matmul8(var_mul39: T.handle, model_layers_0_mlp_down_proj_weight3: T.Buffer((T.int64(2048), T.int64(16384)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() mul39 = T.match_buffer(var_mul39, (T.int64(1), seq_len, T.int64(16384)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(2048)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(2048), T.int64(16384)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(mul39[v_i0, v_i1, v_k], model_layers_0_mlp_down_proj_weight3[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + mul39[v_i0, v_i1, v_k] * model_layers_0_mlp_down_proj_weight3[v_i2, v_k] @T.prim_func(private=True) def NT_matmul9(var_take1: T.handle, var_model_embed_tokens_weight3: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() take1 = T.match_buffer(var_take1, (T.int64(1), batch_size, T.int64(2048)), "float16") vocab_size = T.int64() model_embed_tokens_weight3 = T.match_buffer(var_model_embed_tokens_weight3, (vocab_size, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), batch_size, vocab_size)) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), batch_size, vocab_size, T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(take1[v_i0, v_i1, v_k], model_embed_tokens_weight3[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", take1[v_i0, v_i1, v_k]) * T.Cast("float32", model_embed_tokens_weight3[v_i2, v_k]) @T.prim_func def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") # with T.block("root"): for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) T.writes(logits[seq_ids[vs], vv]) logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) @T.prim_func def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") logit_bias = T.match_buffer(var_logit_bias, (num_token,)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) T.writes(logits[pos2seq_id[vp], token_ids[vp]]) logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] @T.prim_func def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") penalties = T.match_buffer(var_penalties, (num_seq, 3)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) @T.prim_func def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 256), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 256), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.090168440055560212) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): for tz in T.thread_binding(1, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz * 4 + ty, tx * 4 - 128:tx * 4 - 128 + 260]) T.writes(output[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty, tx * 4:tx * 4 + 4], lse[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((4, 256), "float16", scope="shared") V_smem = T.alloc_buffer((4, 256), "float16", scope="shared") O_allreduce = T.alloc_buffer((1, 4, 256), scope="shared") md_allreduce = T.alloc_buffer((1, 4, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((4,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = 0 bz: T.int32 = fused_by_bz batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec + 128] * T.float16(-1.0), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 3) // 4): tile_start_s: T.int32 = tz * 4 + ty tile_start_g: T.int32 = (iterator + tz) * 4 + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = row_g page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, pages[page_no, 0, by, page_offset, tx * 4 + vec + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(4): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz * 4 + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if (iterator + tz) * 4 + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(4): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(4): for vec in T.vectorized(4): V_local[vec] = V_smem[tz * 4 + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by * 8 + bz * 4 + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by * 8 + bz * 4 + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 256), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 256), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.090168440055560212) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): for tz in T.thread_binding(1, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz * 4 + ty, tx * 4 - 128:tx * 4 - 128 + 260]) T.writes(output[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty, tx * 4:tx * 4 + 4], lse[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((4, 256), "float16", scope="shared") V_smem = T.alloc_buffer((4, 256), "float16", scope="shared") O_allreduce = T.alloc_buffer((1, 4, 256), scope="shared") md_allreduce = T.alloc_buffer((1, 4, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((4,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = 0 bz: T.int32 = fused_by_bz batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec + 128] * T.float16(-1.0), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 3) // 4): tile_start_s: T.int32 = tz * 4 + ty tile_start_g: T.int32 = (iterator + tz) * 4 + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, pages[page_no, 0, by, page_offset, tx * 4 + vec + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(4): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz * 4 + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if (iterator + tz) * 4 + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(4): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(4): for vec in T.vectorized(4): V_local[vec] = V_smem[tz * 4 + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by * 8 + bz * 4 + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by * 8 + bz * 4 + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((16, 256), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, pages[page_no, 0, by, page_offset, j + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:16, 0:256]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(1, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 1, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(1, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((16, 256), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, pages[page_no, 0, by, page_offset, j + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:16, 0:256]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(1, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 1, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(1, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 1, 256), "float16") v = T.match_buffer(var_v, (kv_len, 1, 256), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(64, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((256, 16), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] q_indptr_val: T.int32 = q_indptr[b_idx] LH_start: T.int32 = tile_id[0] * 16 kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_0, lj_0_0 in T.grid(2, 2): for lj_1 in T.vectorized(8): with T.block("Q_load"): i = T.axis.spatial(16, li_0 * 8 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 16) j = T.axis.spatial(256, lj_0_0 * 128 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 16 * 8 + lj_1) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(2, 2): for ly_1 in T.vectorized(8): with T.block("K_load"): i = T.axis.spatial(16, lz_0 * 8 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 16) j = T.axis.spatial(256, ly_0_0 * 128 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 16 * 8 + ly_1) T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 128:j - 128 + 257]) T.writes(K_smem[j, i]) cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: freq = T.float32() K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, k[L_kv_base + cur_L, by, j + 128] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), k[L_kv_base + cur_L, by, j]) else: K_smem[j, i] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(2, 2): for ly_1 in T.vectorized(8): with T.block("V_load"): i = T.axis.spatial(16, lz_0 * 8 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 16) j = T.axis.spatial(256, ly_0_0 * 128 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 16 * 8 + ly_1) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: V_smem[i, j] = v[L_kv_base + cur_L, by, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:256, 0:16]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(1): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_0_init * 2 + lj_1_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0 in range(16): for li_1 in T.unroll(1): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(2): for lk_1 in range(16): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1_0 * 2 + lj_1_1) k_1 = T.axis.reduce(256, lk_0 * 16 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(1): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1_0 * 2 + lj_1_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(4): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_0_init * 8 + lj_1_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1 in T.grid(1, 16): for li_1 in T.unroll(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) k_1 = T.axis.reduce(16, lk_0 * 16 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 64 @T.prim_func def batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 256), "float16") q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 1, 256), "float16") v = T.match_buffer(var_v, (kv_len, 1, 256), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) tree_size = T.int32(is_size_var=True) mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((32, 256), "float16", scope="shared") V_smem = T.alloc_buffer((32, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 32), scope="shared") S_local = T.alloc_buffer((16, 32), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("KV_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_base + L_kv_start + i if L_kv_start + i < kv_chunk_len[0]: freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, k[cur_L, by, j + 128] * T.float16(-1.0), k[cur_L, by, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), k[cur_L, by, j]) V_smem[i, j] = v[cur_L, by, j] else: K_smem[i, j] = T.float16(0.0) V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:32, 0:256]) T.writes(S_local[0:16, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) k_1 = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 16: row_: T.int32 = (LH_start + row) // 8 if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:32], V_smem[0:32, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int64(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) # with T.block("root"): A_pad = T.alloc_buffer((batch_size, num_chunks, T.int64(4096))) temp_max = T.alloc_buffer((batch_size, num_chunks)) temp_sum = T.alloc_buffer((batch_size, num_chunks)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("pad"): v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2]) T.writes(A_pad[v0, v1, v2]) A_pad[v0, v1, v2] = T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * T.int64(4096) + v2] / temperature[v0], A[v0, v1 * T.int64(4096) + v2]), T.float32(-340282346638528859811704183484516925440.0)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("max"): v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) T.reads(A_pad[v0, v1, v2]) T.writes(temp_max[v0, v1]) with T.init(): temp_max[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) temp_max[v0, v1] = T.max(temp_max[v0, v1], A_pad[v0, v1, v2]) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("sum_exp"): v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) T.reads(temperature[v0], A_pad[v0, v1, v2], temp_max[v0, v1]) T.writes(temp_sum[v0, v1]) with T.init(): temp_sum[v0, v1] = T.float32(0.0) temp_sum[v0, v1] = temp_sum[v0, v1] + T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A_pad[v0, v1, v2] - temp_max[v0, v1]), T.Cast("float32", A_pad[v0, v1, v2] == temp_max[v0, v1])), T.float32(0.0)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(1)): with T.block("log"): v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) T.reads(temperature[v0], temp_sum[v0, v1], temp_max[v0, v1]) T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum[v0, v1]), temp_sum[v0, v1]) chunked_max[v0, v1] = temp_max[v0, v1] @T.prim_func def compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 1, 16, 256), "float16", offset_factor=1) copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) total_copy_length = T.int32() copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) with T.block("root"): T.reads() T.writes() for bhd_o in T.thread_binding(batch_size, thread="blockIdx.x"): for bhd_i in T.thread_binding(256, thread="threadIdx.x"): b: T.int32 = (bhd_o * 256 + bhd_i) // 256 h: T.int32 = 0 d: T.int32 = (bhd_o * 256 + bhd_i) % 256 if bhd_o * 256 + bhd_i < batch_size * 256: for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] @T.prim_func def copy_single_page(var_pages: T.handle, src_page_id: T.int64, tgt_page_id: T.int64, copy_length: T.int64): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages, page_size = T.int32(), T.int64() pages = T.match_buffer(var_pages, (num_pages, 2, 1, page_size, 256), "float16", offset_factor=1) # with T.block("root"): for b in T.thread_binding(copy_length, thread="blockIdx.x"): for t in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy"): vh = T.axis.spatial(1, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) // (copy_length * T.int64(256)))) vp = T.axis.spatial(copy_length, (b * T.int64(256) + T.Cast("int64", t)) % (copy_length * T.int64(256)) // T.int64(256)) vd = T.axis.spatial(256, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) % T.int64(256))) T.where(b * T.int64(256) + T.Cast("int64", t) < copy_length * T.int64(256)) T.reads(pages[src_page_id, 0:2, vh, vp, vd]) T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] @T.prim_func(private=True) def fuse_add_norm_decode(pA: T.handle, pB: T.handle, C: T.Buffer((2048,), "float16"), pO: T.handle, pAdd: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() A = T.match_buffer(pA, (batch_size, 1, 2048), "float16") B = T.match_buffer(pB, (batch_size, 1, 2048), "float16") O = T.match_buffer(pO, (batch_size, 1, 2048), "float16") add = T.match_buffer(pAdd, (batch_size, 1, 2048), "float16") # with T.block("root"): add_local = T.alloc_buffer((8,), "float16", scope="local") sum_shared = T.alloc_buffer((batch_size, 1), scope="shared") sum_local = T.alloc_buffer((256, batch_size, 1), scope="local") for v_bx in T.thread_binding(batch_size, thread="blockIdx.x"): for v_tx in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): for i in range(8): with T.block("T_add"): bx = T.axis.spatial(batch_size, v_bx) h = T.axis.spatial(2048, i * 256 + v_tx) T.reads(A[bx, 0, h], B[bx, 0, h]) T.writes(add_local[h // 256]) add_local[h // 256] = A[bx, 0, h] + B[bx, 0, h] with T.block("T_write_back"): bx = T.axis.spatial(batch_size, v_bx) v_ax1 = T.axis.spatial(1, 0) h = T.axis.spatial(2048, i * 256 + v_tx) T.reads(add_local[h // 256]) T.writes(add[bx, v_ax1, h]) add[bx, v_ax1, h] = add_local[h // 256] with T.block("T_multiply_red_rf_init"): tx, bx = T.axis.remap("SS", [v_tx, v_bx]) T.reads() T.writes(sum_local[tx, bx, 0]) sum_local[tx, bx, 0] = T.float32(0.0) for v_i, _j in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): tx, bx, i = T.axis.remap("SSR", [v_tx, v_bx, v_i]) T.reads(sum_local[tx, bx, 0], add_local[i]) T.writes(sum_local[tx, bx, 0]) sum_local[tx, bx, 0] = sum_local[tx, bx, 0] + T.Cast("float32", add_local[i]) * T.Cast("float32", add_local[i]) for _j in range(1): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): tx, bx = T.axis.remap("RS", [v_tx_2, v_bx]) T.reads(sum_local[tx, bx, 0]) T.writes(sum_shared[bx, 0]) with T.init(): sum_shared[bx, 0] = T.float32(0.0) sum_shared[bx, 0] = sum_shared[bx, 0] + sum_local[tx, bx, 0] for i in range(8): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): bx = T.axis.spatial(batch_size, v_bx) h = T.axis.spatial(2048, i * 256 + v_tx_2) T.reads(sum_shared[bx, 0], add_local[h // 256], C[h]) T.writes(O[bx, 0, h]) O[bx, 0, h] = T.Cast("float16", T.rsqrt(sum_shared[bx, 0] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", add_local[h // 256]) * T.Cast("float32", C[h])) @T.prim_func(private=True) def fuse_add_norm_prefill(pA: T.handle, pB: T.handle, C: T.Buffer((2048,), "float16"), pO: T.handle, pAdd: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() A = T.match_buffer(pA, (1, seq_len, 2048), "float16") B = T.match_buffer(pB, (1, seq_len, 2048), "float16") O = T.match_buffer(pO, (1, seq_len, 2048), "float16") add = T.match_buffer(pAdd, (1, seq_len, 2048), "float16") # with T.block("root"): add_local = T.alloc_buffer((8,), "float16", scope="local") sum_shared = T.alloc_buffer((1, seq_len), scope="shared") sum_local = T.alloc_buffer((256, 1, seq_len), scope="local") for v_bx in T.thread_binding(seq_len, thread="blockIdx.x"): for v_tx in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): for v_i in range(8): with T.block("T_add"): bx = T.axis.spatial(seq_len, v_bx) h = T.axis.spatial(2048, v_i * 256 + v_tx) T.reads(A[0, bx, h], B[0, bx, h]) T.writes(add_local[h // 256]) add_local[h // 256] = A[0, bx, h] + B[0, bx, h] with T.block("T_write_back"): bx = T.axis.spatial(seq_len, v_bx) h = T.axis.spatial(2048, v_i * 256 + v_tx) T.reads(add_local[h // 256]) T.writes(add[0, bx, h]) add[0, bx, h] = add_local[h // 256] with T.block("T_multiply_red_rf_init"): tx, bx = T.axis.remap("SS", [v_tx, v_bx]) T.reads() T.writes(sum_local[tx, 0, bx]) sum_local[tx, 0, bx] = T.float32(0.0) for v_i, _j in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): tx, bx, i = T.axis.remap("SSR", [v_tx, v_bx, v_i]) T.reads(sum_local[tx, 0, bx], add_local[i]) T.writes(sum_local[tx, 0, bx]) sum_local[tx, 0, bx] = sum_local[tx, 0, bx] + T.Cast("float32", add_local[i]) * T.Cast("float32", add_local[i]) for _j in range(1): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): tx, bx = T.axis.remap("RS", [v_tx_2, v_bx]) T.reads(sum_local[tx, 0, bx]) T.writes(sum_shared[0, bx]) with T.init(): sum_shared[0, bx] = T.float32(0.0) sum_shared[0, bx] = sum_shared[0, bx] + sum_local[tx, 0, bx] for v_i in range(8): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): bx = T.axis.spatial(seq_len, v_bx) v1 = T.axis.spatial(2048, v_i * 256 + v_tx_2) T.reads(sum_shared[0, bx], add_local[v1 // 256], C[v1]) T.writes(O[0, bx, v1]) O[0, bx, v1] = T.Cast("float16", T.rsqrt(sum_shared[0, bx] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", add_local[v1 // 256]) * T.Cast("float32", C[v1])) @T.prim_func def fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) seq_len = T.int32() qkv = T.match_buffer(var_qkv, (seq_len, 10, 256), "float16") position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) q = T.match_buffer(var_q, (seq_len, 8, 256), "float16") k = T.match_buffer(var_k, (seq_len, 1, 256), "float16") v = T.match_buffer(var_v, (seq_len, 1, 256), "float16") # with T.block("root"): for iters_0, iters_1, iters_2 in T.grid(seq_len, 10, 256): with T.block("llama_fused_rope"): s, h, d = T.axis.remap("SSS", [iters_0, iters_1, iters_2]) T.reads(position_map[s], qkv[s, h, d - 128:d - 128 + 257]) T.writes(q[s, h, d], k[s, h - 8, d], v[s, h - 9, d]) if h < 8: freq = T.float32() q[s, h, d] = T.if_then_else(apply_rope > 0 and d < 256, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 128, qkv[s, h, d + 128] * T.float16(-1.0), qkv[s, h, d - 128]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 256) / T.float32(256.0))}), qkv[s, h, d]) else: if h < 9: freq = T.float32() k[s, h - 8, d] = T.if_then_else(apply_rope > 0 and d < 256, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 128, qkv[s, h, d + 128] * T.float16(-1.0), qkv[s, h, d - 128]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 256) / T.float32(256.0))}), qkv[s, h, d]) else: v[s, h - 9, d] = qkv[s, h, d] @T.prim_func def gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (m, n)) batch_size = T.int32(is_size_var=True) indices = T.match_buffer(var_indices, (batch_size,), "int32") dst = T.match_buffer(var_dst, (batch_size, n)) # with T.block("root"): for b, j in T.grid(batch_size, n): with T.block("gather_2d"): vb, vj = T.axis.remap("SS", [b, j]) T.reads(src[indices[vb], vj], indices[vb]) T.writes(dst[vb, vj]) dst[vb, vj] = src[indices[vb], vj] @T.prim_func(private=True) def index(var_rms_norm36: T.handle, index: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) seq_len = T.int64() rms_norm36 = T.match_buffer(var_rms_norm36, (T.int64(1), seq_len, T.int64(2048)), "float16") # with T.block("root"): for i, _, k in T.grid(T.int64(1), T.int64(1), T.int64(2048)): with T.block("index"): v_i, v__, v_k = T.axis.remap("SSS", [i, _, k]) T.reads(rms_norm36[v_i, seq_len - T.int64(1), v_k]) T.writes(index[v_i, v__, v_k]) index[v_i, v__, v_k] = rms_norm36[v_i, seq_len - T.int64(1), v_k] @T.prim_func def merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) V = T.match_buffer(v, (N, H, D), "float16") S = T.match_buffer(s, (N, H)) V_other = T.match_buffer(v_other, (N, H, D), "float16") S_other = T.match_buffer(s_other, (N, H)) # with T.block("root"): for bx in T.thread_binding(N, thread="blockIdx.x"): for by in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): with T.block("merge"): T.reads(S[bx, ty + by * 4], S_other[bx, ty + by * 4], V[bx, ty + by * 4, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 4, tx * 4:tx * 4 + 4]) T.writes(V[bx, ty + by * 4, tx * 4:tx * 4 + 4], S[bx, ty + by * 4]) s_val = T.alloc_buffer((1,), scope="local") s_other_val = T.alloc_buffer((1,), scope="local") s_max = T.alloc_buffer((1,), scope="local") scale = T.alloc_buffer((1,), scope="local") other_scale = T.alloc_buffer((1,), scope="local") v_vec = T.alloc_buffer((4,), "float16", scope="local") v_other_vec = T.alloc_buffer((4,), "float16", scope="local") s_val[0] = S[bx, ty + by * 4] s_other_val[0] = S_other[bx, ty + by * 4] s_max[0] = T.max(s_val[0], s_other_val[0]) s_val[0] = T.exp2(s_val[0] - s_max[0]) s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) for vec in T.vectorized(4): v_vec[vec] = V[bx, ty + by * 4, tx * 4 + vec] for vec in T.vectorized(4): v_other_vec[vec] = V_other[bx, ty + by * 4, tx * 4 + vec] for vec in range(4): v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) for vec in T.vectorized(4): V[bx, ty + by * 4, tx * 4 + vec] = v_vec[vec] S[bx, ty + by * 4] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] @T.prim_func def scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (batch_size, n)) indices = T.match_buffer(var_indices, (batch_size,), "int32") m = T.int32(is_size_var=True) dst = T.match_buffer(var_dst, (m, n)) # with T.block("root"): for b, j in T.grid(batch_size, n): with T.block("scatter_2d"): vb, vj = T.axis.remap("SS", [b, j]) T.reads(src[vb, vj], indices[vb]) T.writes(dst[indices[vb], vj]) dst[indices[vb], vj] = src[vb, vj] @T.prim_func def softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int64(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) T.reads(chunked_max[v0, v1]) T.writes(temp_max_shared[v0]) with T.init(): temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) T.writes(temp_sum_shared[v0]) with T.init(): temp_sum_shared[v0] = T.float32(0.0) temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) for l2_0 in T.serial(T.int64(16), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): for l2_1 in T.thread_binding(T.int64(8), thread="threadIdx.y"): for l2_2 in T.thread_binding(T.int64(32), thread="threadIdx.x"): with T.block("log_pad"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) v2 = T.axis.spatial(T.int64(4096), l2_0 * T.int64(256) + l2_1 * T.int64(32) + l2_2) T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2], temp_sum_shared[v0], temp_max_shared[v0]) T.writes(softmax[v0, v1 * T.int64(4096) + v2]) if v1 * T.int64(4096) + v2 < vocab_size: softmax[v0, v1 * T.int64(4096) + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * T.int64(4096) + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * T.int64(4096) + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) @T.prim_func def tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int64): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) num_pages, page_size = T.int64(), T.int64(is_size_var=True) pages = T.match_buffer(var_pages, (num_pages, 2, 1, page_size, 256), "float16", offset_factor=1) seqlen = T.int64(is_size_var=True) position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) k_data = T.match_buffer(var_k_data, (18, seqlen, 1, 256), "float16") v_data = T.match_buffer(var_v_data, (18, seqlen, 1, 256), "float16") # with T.block("root"): for p, h, d in T.grid(seqlen, 1, 256): with T.block("copy0"): vp, vh, vd = T.axis.remap("SSS", [p, h, d]) T.reads(position_map[vp], pages[T.Cast("int64", position_map[vp]) // page_size, 0:2, vh, T.Cast("int64", position_map[vp]) % page_size, vd]) T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) position: T.int32 = position_map[vp] k_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 0, vh, T.Cast("int64", position) % page_size, vd] v_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 1, vh, T.Cast("int64", position) % page_size, vd] @T.prim_func def tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) num_pages = T.int64() pages = T.match_buffer(var_pages, (num_pages, 2, 1, 16, 256), "float16", offset_factor=1) ntoken = T.int64(is_size_var=True) k_data = T.match_buffer(var_k_data, (ntoken, 1, 256), "float16") v_data = T.match_buffer(var_v_data, (ntoken, 1, 256), "float16") position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) # with T.block("root"): for global_pos, h, f in T.grid(ntoken, 1, 256): if position_map[global_pos] != -1: with T.block("k_transpose_append"): vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] with T.block("v_transpose_append"): vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] @T.prim_func def tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 32, 256), "float16") page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) total_tree_order_len = T.int32(is_size_var=True) tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) # with T.block("root"): assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((32, 256), "float16", scope="shared") V_smem = T.alloc_buffer((32, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 32), scope="shared") S_local = T.alloc_buffer((16, 32), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 K_smem[i, j] = pages[page_no, 0, by, page_offset, j] else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:32, 0:256]) T.writes(S_local[0:16, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 16: row_: T.int32 = (LH_start + row) // 8 if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:32], V_smem[0:32, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @R.function def alloc_embedding_tensor() -> R.Tensor((8192, 2048), dtype="float16"): R.func_attr({"relax.memory_plan_dynamic_func_output": True}) gv: R.Tensor((8192, 2048), dtype="float16") = R.builtin.alloc_tensor(R.shape([8192, 2048]), R.dtype("float16"), R.prim_value(0), R.str("global")) return gv @R.function def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): model_embed_tokens_weight4: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[109] mul57: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.multiply(input_embeds, R.const(45.25, "float16")) rms_norm111: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.nn.rms_norm(mul57, model_layers_0_input_layernorm_weight4, axes=[-1], epsilon=9.9999999999999995e-07) lv = R.call_tir(cls.NT_matmul, (rms_norm111, model_layers_0_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape216: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv, R.shape([batch_size, 1, 10, 256])) reshape217: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape216, R.shape([batch_size, 10, 256])) lv55 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape217), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape218: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv55, R.shape([batch_size, 1, 8, 256])) reshape219: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape218, R.shape([batch_size, 1, 2048])) lv1 = R.call_tir(cls.NT_matmul1, (reshape219, model_layers_0_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv_1 = R.call_tir(cls.fuse_add_norm_decode, (lv1, mul57, model_layers_0_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv1_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv_1[1] rms_norm112: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv_1[0] lv2 = R.call_tir(cls.NT_matmul2, (rms_norm112, model_layers_0_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split54: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv2, indices_or_sections=2, axis=-1) split_054: R.Tensor((batch_size, 1, 16384), dtype="float16") = split54[0] split_154: R.Tensor((batch_size, 1, 16384), dtype="float16") = split54[1] gelu54: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_054) mul58: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu54, split_154) lv3 = R.call_tir(cls.NT_matmul3, (mul58, model_layers_0_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv2_1 = R.call_tir(cls.fuse_add_norm_decode, (lv3, lv1_1, model_layers_1_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv3_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv2_1[1] rms_norm113: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv2_1[0] lv4 = R.call_tir(cls.NT_matmul, (rms_norm113, model_layers_1_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape220: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv4, R.shape([batch_size, 1, 10, 256])) reshape221: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape220, R.shape([batch_size, 10, 256])) lv56 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape221), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape222: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv56, R.shape([batch_size, 1, 8, 256])) reshape223: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape222, R.shape([batch_size, 1, 2048])) lv5 = R.call_tir(cls.NT_matmul1, (reshape223, model_layers_1_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv4_1 = R.call_tir(cls.fuse_add_norm_decode, (lv5, lv3_1, model_layers_1_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv5_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv4_1[1] rms_norm114: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv4_1[0] lv6 = R.call_tir(cls.NT_matmul2, (rms_norm114, model_layers_1_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split55: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv6, indices_or_sections=2, axis=-1) split_055: R.Tensor((batch_size, 1, 16384), dtype="float16") = split55[0] split_155: R.Tensor((batch_size, 1, 16384), dtype="float16") = split55[1] gelu55: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_055) mul59: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu55, split_155) lv7 = R.call_tir(cls.NT_matmul3, (mul59, model_layers_1_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv6_1 = R.call_tir(cls.fuse_add_norm_decode, (lv7, lv5_1, model_layers_2_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv7_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv6_1[1] rms_norm115: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv6_1[0] lv8 = R.call_tir(cls.NT_matmul, (rms_norm115, model_layers_2_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape224: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv8, R.shape([batch_size, 1, 10, 256])) reshape225: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape224, R.shape([batch_size, 10, 256])) lv57 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape225), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape226: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv57, R.shape([batch_size, 1, 8, 256])) reshape227: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape226, R.shape([batch_size, 1, 2048])) lv9 = R.call_tir(cls.NT_matmul1, (reshape227, model_layers_2_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv8_1 = R.call_tir(cls.fuse_add_norm_decode, (lv9, lv7_1, model_layers_2_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv9_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv8_1[1] rms_norm116: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv8_1[0] lv10 = R.call_tir(cls.NT_matmul2, (rms_norm116, model_layers_2_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split56: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv10, indices_or_sections=2, axis=-1) split_056: R.Tensor((batch_size, 1, 16384), dtype="float16") = split56[0] split_156: R.Tensor((batch_size, 1, 16384), dtype="float16") = split56[1] gelu56: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_056) mul60: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu56, split_156) lv11 = R.call_tir(cls.NT_matmul3, (mul60, model_layers_2_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv10_1 = R.call_tir(cls.fuse_add_norm_decode, (lv11, lv9_1, model_layers_3_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv11_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv10_1[1] rms_norm117: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv10_1[0] lv12 = R.call_tir(cls.NT_matmul, (rms_norm117, model_layers_3_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape228: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv12, R.shape([batch_size, 1, 10, 256])) reshape229: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape228, R.shape([batch_size, 10, 256])) lv58 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape229), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape230: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv58, R.shape([batch_size, 1, 8, 256])) reshape231: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape230, R.shape([batch_size, 1, 2048])) lv13 = R.call_tir(cls.NT_matmul1, (reshape231, model_layers_3_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv12_1 = R.call_tir(cls.fuse_add_norm_decode, (lv13, lv11_1, model_layers_3_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv13_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv12_1[1] rms_norm118: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv12_1[0] lv14 = R.call_tir(cls.NT_matmul2, (rms_norm118, model_layers_3_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split57: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv14, indices_or_sections=2, axis=-1) split_057: R.Tensor((batch_size, 1, 16384), dtype="float16") = split57[0] split_157: R.Tensor((batch_size, 1, 16384), dtype="float16") = split57[1] gelu57: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_057) mul61: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu57, split_157) lv15 = R.call_tir(cls.NT_matmul3, (mul61, model_layers_3_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv14_1 = R.call_tir(cls.fuse_add_norm_decode, (lv15, lv13_1, model_layers_4_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv15_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv14_1[1] rms_norm119: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv14_1[0] lv16 = R.call_tir(cls.NT_matmul, (rms_norm119, model_layers_4_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape232: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv16, R.shape([batch_size, 1, 10, 256])) reshape233: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape232, R.shape([batch_size, 10, 256])) lv59 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape233), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape234: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv59, R.shape([batch_size, 1, 8, 256])) reshape235: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape234, R.shape([batch_size, 1, 2048])) lv17 = R.call_tir(cls.NT_matmul1, (reshape235, model_layers_4_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv16_1 = R.call_tir(cls.fuse_add_norm_decode, (lv17, lv15_1, model_layers_4_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv17_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv16_1[1] rms_norm120: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv16_1[0] lv18 = R.call_tir(cls.NT_matmul2, (rms_norm120, model_layers_4_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split58: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv18, indices_or_sections=2, axis=-1) split_058: R.Tensor((batch_size, 1, 16384), dtype="float16") = split58[0] split_158: R.Tensor((batch_size, 1, 16384), dtype="float16") = split58[1] gelu58: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_058) mul62: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu58, split_158) lv19 = R.call_tir(cls.NT_matmul3, (mul62, model_layers_4_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv18_1 = R.call_tir(cls.fuse_add_norm_decode, (lv19, lv17_1, model_layers_5_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv19_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv18_1[1] rms_norm121: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv18_1[0] lv20 = R.call_tir(cls.NT_matmul, (rms_norm121, model_layers_5_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape236: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv20, R.shape([batch_size, 1, 10, 256])) reshape237: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape236, R.shape([batch_size, 10, 256])) lv60 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape237), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape238: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv60, R.shape([batch_size, 1, 8, 256])) reshape239: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape238, R.shape([batch_size, 1, 2048])) lv21 = R.call_tir(cls.NT_matmul1, (reshape239, model_layers_5_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv20_1 = R.call_tir(cls.fuse_add_norm_decode, (lv21, lv19_1, model_layers_5_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv21_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv20_1[1] rms_norm122: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv20_1[0] lv22 = R.call_tir(cls.NT_matmul2, (rms_norm122, model_layers_5_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split59: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv22, indices_or_sections=2, axis=-1) split_059: R.Tensor((batch_size, 1, 16384), dtype="float16") = split59[0] split_159: R.Tensor((batch_size, 1, 16384), dtype="float16") = split59[1] gelu59: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_059) mul63: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu59, split_159) lv23 = R.call_tir(cls.NT_matmul3, (mul63, model_layers_5_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv22_1 = R.call_tir(cls.fuse_add_norm_decode, (lv23, lv21_1, model_layers_6_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv23_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv22_1[1] rms_norm123: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv22_1[0] lv24 = R.call_tir(cls.NT_matmul, (rms_norm123, model_layers_6_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape240: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv24, R.shape([batch_size, 1, 10, 256])) reshape241: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape240, R.shape([batch_size, 10, 256])) lv61 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape241), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape242: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv61, R.shape([batch_size, 1, 8, 256])) reshape243: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape242, R.shape([batch_size, 1, 2048])) lv25 = R.call_tir(cls.NT_matmul1, (reshape243, model_layers_6_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv24_1 = R.call_tir(cls.fuse_add_norm_decode, (lv25, lv23_1, model_layers_6_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv25_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv24_1[1] rms_norm124: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv24_1[0] lv26 = R.call_tir(cls.NT_matmul2, (rms_norm124, model_layers_6_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split60: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv26, indices_or_sections=2, axis=-1) split_060: R.Tensor((batch_size, 1, 16384), dtype="float16") = split60[0] split_160: R.Tensor((batch_size, 1, 16384), dtype="float16") = split60[1] gelu60: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_060) mul64: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu60, split_160) lv27 = R.call_tir(cls.NT_matmul3, (mul64, model_layers_6_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv26_1 = R.call_tir(cls.fuse_add_norm_decode, (lv27, lv25_1, model_layers_7_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv27_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv26_1[1] rms_norm125: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv26_1[0] lv28 = R.call_tir(cls.NT_matmul, (rms_norm125, model_layers_7_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape244: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv28, R.shape([batch_size, 1, 10, 256])) reshape245: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape244, R.shape([batch_size, 10, 256])) lv62 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape245), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape246: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv62, R.shape([batch_size, 1, 8, 256])) reshape247: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape246, R.shape([batch_size, 1, 2048])) lv29 = R.call_tir(cls.NT_matmul1, (reshape247, model_layers_7_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv28_1 = R.call_tir(cls.fuse_add_norm_decode, (lv29, lv27_1, model_layers_7_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv29_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv28_1[1] rms_norm126: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv28_1[0] lv30 = R.call_tir(cls.NT_matmul2, (rms_norm126, model_layers_7_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split61: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv30, indices_or_sections=2, axis=-1) split_061: R.Tensor((batch_size, 1, 16384), dtype="float16") = split61[0] split_161: R.Tensor((batch_size, 1, 16384), dtype="float16") = split61[1] gelu61: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_061) mul65: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu61, split_161) lv31 = R.call_tir(cls.NT_matmul3, (mul65, model_layers_7_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv30_1 = R.call_tir(cls.fuse_add_norm_decode, (lv31, lv29_1, model_layers_8_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv31_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv30_1[1] rms_norm127: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv30_1[0] lv32 = R.call_tir(cls.NT_matmul, (rms_norm127, model_layers_8_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape248: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv32, R.shape([batch_size, 1, 10, 256])) reshape249: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape248, R.shape([batch_size, 10, 256])) lv63 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape249), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape250: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv63, R.shape([batch_size, 1, 8, 256])) reshape251: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape250, R.shape([batch_size, 1, 2048])) lv33 = R.call_tir(cls.NT_matmul1, (reshape251, model_layers_8_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv32_1 = R.call_tir(cls.fuse_add_norm_decode, (lv33, lv31_1, model_layers_8_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv33_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv32_1[1] rms_norm128: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv32_1[0] lv34 = R.call_tir(cls.NT_matmul2, (rms_norm128, model_layers_8_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split62: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv34, indices_or_sections=2, axis=-1) split_062: R.Tensor((batch_size, 1, 16384), dtype="float16") = split62[0] split_162: R.Tensor((batch_size, 1, 16384), dtype="float16") = split62[1] gelu62: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_062) mul66: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu62, split_162) lv35 = R.call_tir(cls.NT_matmul3, (mul66, model_layers_8_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv34_1 = R.call_tir(cls.fuse_add_norm_decode, (lv35, lv33_1, model_layers_9_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv35_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv34_1[1] rms_norm129: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv34_1[0] lv36 = R.call_tir(cls.NT_matmul, (rms_norm129, model_layers_9_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape252: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv36, R.shape([batch_size, 1, 10, 256])) reshape253: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape252, R.shape([batch_size, 10, 256])) lv64 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape253), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape254: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv64, R.shape([batch_size, 1, 8, 256])) reshape255: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape254, R.shape([batch_size, 1, 2048])) lv37 = R.call_tir(cls.NT_matmul1, (reshape255, model_layers_9_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv36_1 = R.call_tir(cls.fuse_add_norm_decode, (lv37, lv35_1, model_layers_9_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv37_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv36_1[1] rms_norm130: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv36_1[0] lv38 = R.call_tir(cls.NT_matmul2, (rms_norm130, model_layers_9_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split63: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv38, indices_or_sections=2, axis=-1) split_063: R.Tensor((batch_size, 1, 16384), dtype="float16") = split63[0] split_163: R.Tensor((batch_size, 1, 16384), dtype="float16") = split63[1] gelu63: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_063) mul67: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu63, split_163) lv39 = R.call_tir(cls.NT_matmul3, (mul67, model_layers_9_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv38_1 = R.call_tir(cls.fuse_add_norm_decode, (lv39, lv37_1, model_layers_10_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv39_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv38_1[1] rms_norm131: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv38_1[0] lv40 = R.call_tir(cls.NT_matmul, (rms_norm131, model_layers_10_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape256: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv40, R.shape([batch_size, 1, 10, 256])) reshape257: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape256, R.shape([batch_size, 10, 256])) lv65 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape257), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape258: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv65, R.shape([batch_size, 1, 8, 256])) reshape259: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape258, R.shape([batch_size, 1, 2048])) lv41 = R.call_tir(cls.NT_matmul1, (reshape259, model_layers_10_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv40_1 = R.call_tir(cls.fuse_add_norm_decode, (lv41, lv39_1, model_layers_10_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv41_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv40_1[1] rms_norm132: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv40_1[0] lv42 = R.call_tir(cls.NT_matmul2, (rms_norm132, model_layers_10_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split64: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv42, indices_or_sections=2, axis=-1) split_064: R.Tensor((batch_size, 1, 16384), dtype="float16") = split64[0] split_164: R.Tensor((batch_size, 1, 16384), dtype="float16") = split64[1] gelu64: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_064) mul68: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu64, split_164) lv43 = R.call_tir(cls.NT_matmul3, (mul68, model_layers_10_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv42_1 = R.call_tir(cls.fuse_add_norm_decode, (lv43, lv41_1, model_layers_11_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv43_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv42_1[1] rms_norm133: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv42_1[0] lv44 = R.call_tir(cls.NT_matmul, (rms_norm133, model_layers_11_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape260: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv44, R.shape([batch_size, 1, 10, 256])) reshape261: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape260, R.shape([batch_size, 10, 256])) lv66 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape261), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape262: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv66, R.shape([batch_size, 1, 8, 256])) reshape263: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape262, R.shape([batch_size, 1, 2048])) lv45 = R.call_tir(cls.NT_matmul1, (reshape263, model_layers_11_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv44_1 = R.call_tir(cls.fuse_add_norm_decode, (lv45, lv43_1, model_layers_11_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv45_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv44_1[1] rms_norm134: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv44_1[0] lv46 = R.call_tir(cls.NT_matmul2, (rms_norm134, model_layers_11_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split65: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv46, indices_or_sections=2, axis=-1) split_065: R.Tensor((batch_size, 1, 16384), dtype="float16") = split65[0] split_165: R.Tensor((batch_size, 1, 16384), dtype="float16") = split65[1] gelu65: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_065) mul69: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu65, split_165) lv47 = R.call_tir(cls.NT_matmul3, (mul69, model_layers_11_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv46_1 = R.call_tir(cls.fuse_add_norm_decode, (lv47, lv45_1, model_layers_12_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv47_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv46_1[1] rms_norm135: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv46_1[0] lv48 = R.call_tir(cls.NT_matmul, (rms_norm135, model_layers_12_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape264: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv48, R.shape([batch_size, 1, 10, 256])) reshape265: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape264, R.shape([batch_size, 10, 256])) lv67 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape265), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape266: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv67, R.shape([batch_size, 1, 8, 256])) reshape267: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape266, R.shape([batch_size, 1, 2048])) lv49 = R.call_tir(cls.NT_matmul1, (reshape267, model_layers_12_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv48_1 = R.call_tir(cls.fuse_add_norm_decode, (lv49, lv47_1, model_layers_12_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv49_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv48_1[1] rms_norm136: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv48_1[0] lv50 = R.call_tir(cls.NT_matmul2, (rms_norm136, model_layers_12_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split66: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv50, indices_or_sections=2, axis=-1) split_066: R.Tensor((batch_size, 1, 16384), dtype="float16") = split66[0] split_166: R.Tensor((batch_size, 1, 16384), dtype="float16") = split66[1] gelu66: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_066) mul70: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu66, split_166) lv51 = R.call_tir(cls.NT_matmul3, (mul70, model_layers_12_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv50_1 = R.call_tir(cls.fuse_add_norm_decode, (lv51, lv49_1, model_layers_13_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv51_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv50_1[1] rms_norm137: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv50_1[0] lv52 = R.call_tir(cls.NT_matmul, (rms_norm137, model_layers_13_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape268: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv52, R.shape([batch_size, 1, 10, 256])) reshape269: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape268, R.shape([batch_size, 10, 256])) lv68 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape269), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape270: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv68, R.shape([batch_size, 1, 8, 256])) reshape271: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape270, R.shape([batch_size, 1, 2048])) lv53 = R.call_tir(cls.NT_matmul1, (reshape271, model_layers_13_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv52_1 = R.call_tir(cls.fuse_add_norm_decode, (lv53, lv51_1, model_layers_13_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv53_1: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv52_1[1] rms_norm138: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv52_1[0] lv54 = R.call_tir(cls.NT_matmul2, (rms_norm138, model_layers_13_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split67: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv54, indices_or_sections=2, axis=-1) split_067: R.Tensor((batch_size, 1, 16384), dtype="float16") = split67[0] split_167: R.Tensor((batch_size, 1, 16384), dtype="float16") = split67[1] gelu67: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_067) mul71: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu67, split_167) lv55_1 = R.call_tir(cls.NT_matmul3, (mul71, model_layers_13_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv54_1 = R.call_tir(cls.fuse_add_norm_decode, (lv55_1, lv53_1, model_layers_14_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv55_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv54_1[1] rms_norm139: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv54_1[0] lv56_1 = R.call_tir(cls.NT_matmul, (rms_norm139, model_layers_14_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape272: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv56_1, R.shape([batch_size, 1, 10, 256])) reshape273: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape272, R.shape([batch_size, 10, 256])) lv69 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape273), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape274: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv69, R.shape([batch_size, 1, 8, 256])) reshape275: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape274, R.shape([batch_size, 1, 2048])) lv57_1 = R.call_tir(cls.NT_matmul1, (reshape275, model_layers_14_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv56_2 = R.call_tir(cls.fuse_add_norm_decode, (lv57_1, lv55_2, model_layers_14_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv57_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv56_2[1] rms_norm140: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv56_2[0] lv58_1 = R.call_tir(cls.NT_matmul2, (rms_norm140, model_layers_14_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split68: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv58_1, indices_or_sections=2, axis=-1) split_068: R.Tensor((batch_size, 1, 16384), dtype="float16") = split68[0] split_168: R.Tensor((batch_size, 1, 16384), dtype="float16") = split68[1] gelu68: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_068) mul72: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu68, split_168) lv59_1 = R.call_tir(cls.NT_matmul3, (mul72, model_layers_14_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv58_2 = R.call_tir(cls.fuse_add_norm_decode, (lv59_1, lv57_2, model_layers_15_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv59_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv58_2[1] rms_norm141: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv58_2[0] lv60_1 = R.call_tir(cls.NT_matmul, (rms_norm141, model_layers_15_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape276: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv60_1, R.shape([batch_size, 1, 10, 256])) reshape277: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape276, R.shape([batch_size, 10, 256])) lv70 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape277), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape278: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv70, R.shape([batch_size, 1, 8, 256])) reshape279: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape278, R.shape([batch_size, 1, 2048])) lv61_1 = R.call_tir(cls.NT_matmul1, (reshape279, model_layers_15_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv60_2 = R.call_tir(cls.fuse_add_norm_decode, (lv61_1, lv59_2, model_layers_15_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv61_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv60_2[1] rms_norm142: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv60_2[0] lv62_1 = R.call_tir(cls.NT_matmul2, (rms_norm142, model_layers_15_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split69: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv62_1, indices_or_sections=2, axis=-1) split_069: R.Tensor((batch_size, 1, 16384), dtype="float16") = split69[0] split_169: R.Tensor((batch_size, 1, 16384), dtype="float16") = split69[1] gelu69: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_069) mul73: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu69, split_169) lv63_1 = R.call_tir(cls.NT_matmul3, (mul73, model_layers_15_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv62_2 = R.call_tir(cls.fuse_add_norm_decode, (lv63_1, lv61_2, model_layers_16_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv63_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv62_2[1] rms_norm143: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv62_2[0] lv64_1 = R.call_tir(cls.NT_matmul, (rms_norm143, model_layers_16_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape280: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv64_1, R.shape([batch_size, 1, 10, 256])) reshape281: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape280, R.shape([batch_size, 10, 256])) lv71 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape281), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape282: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv71, R.shape([batch_size, 1, 8, 256])) reshape283: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape282, R.shape([batch_size, 1, 2048])) lv65_1 = R.call_tir(cls.NT_matmul1, (reshape283, model_layers_16_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv64_2 = R.call_tir(cls.fuse_add_norm_decode, (lv65_1, lv63_2, model_layers_16_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv65_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv64_2[1] rms_norm144: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv64_2[0] lv66_1 = R.call_tir(cls.NT_matmul2, (rms_norm144, model_layers_16_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split70: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv66_1, indices_or_sections=2, axis=-1) split_070: R.Tensor((batch_size, 1, 16384), dtype="float16") = split70[0] split_170: R.Tensor((batch_size, 1, 16384), dtype="float16") = split70[1] gelu70: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_070) mul74: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu70, split_170) lv67_1 = R.call_tir(cls.NT_matmul3, (mul74, model_layers_16_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv66_2 = R.call_tir(cls.fuse_add_norm_decode, (lv67_1, lv65_2, model_layers_17_input_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv67_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv66_2[1] rms_norm145: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv66_2[0] lv68_1 = R.call_tir(cls.NT_matmul, (rms_norm145, model_layers_17_self_attn_qkv_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2560), dtype="float16")) reshape284: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.reshape(lv68_1, R.shape([batch_size, 1, 10, 256])) reshape285: R.Tensor((batch_size, 10, 256), dtype="float16") = R.reshape(reshape284, R.shape([batch_size, 10, 256])) lv72 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape285), out_sinfo=R.Tensor((batch_size, 8, 256), dtype="float16")) reshape286: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.reshape(lv72, R.shape([batch_size, 1, 8, 256])) reshape287: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.reshape(reshape286, R.shape([batch_size, 1, 2048])) lv69_1 = R.call_tir(cls.NT_matmul1, (reshape287, model_layers_17_self_attn_o_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv68_2 = R.call_tir(cls.fuse_add_norm_decode, (lv69_1, lv67_2, model_layers_17_post_attention_layernorm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) lv69_2: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv68_2[1] rms_norm146: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv68_2[0] lv70_1 = R.call_tir(cls.NT_matmul2, (rms_norm146, model_layers_17_mlp_gate_up_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 32768), dtype="float16")) split71: R.Tuple(R.Tensor((batch_size, 1, 16384), dtype="float16"), R.Tensor((batch_size, 1, 16384), dtype="float16")) = R.split(lv70_1, indices_or_sections=2, axis=-1) split_071: R.Tensor((batch_size, 1, 16384), dtype="float16") = split71[0] split_171: R.Tensor((batch_size, 1, 16384), dtype="float16") = split71[1] gelu71: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_071) mul75: R.Tensor((batch_size, 1, 16384), dtype="float16") = R.multiply(gelu71, split_171) lv71_1 = R.call_tir(cls.NT_matmul3, (mul75, model_layers_17_mlp_down_proj_weight4), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float16")) lv70_2 = R.call_tir(cls.fuse_add_norm_decode, (lv71_1, lv69_2, model_norm_weight4), out_sinfo=[R.Tensor((batch_size, 1, 2048), dtype="float16"), R.Tensor((batch_size, 1, 2048), dtype="float16")]) rms_norm147: R.Tensor((batch_size, 1, 2048), dtype="float16") = lv70_2[0] lv72_1 = R.call_tir(cls.NT_matmul4, (rms_norm147, model_embed_tokens_weight4), out_sinfo=R.Tensor((batch_size, 1, vocab_size), dtype="float32")) gv4: R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype="float32"), R.Object) = lv72_1, paged_kv_cache R.output(gv4) return gv4 @R.function def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): model_embed_tokens_weight3: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[109] mul38: R.Tensor((1, seq_len, 2048), dtype="float16") = R.multiply(input_embeds, R.const(45.25, "float16")) rms_norm74: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(mul38, model_layers_0_input_layernorm_weight3, axes=[-1], epsilon=9.9999999999999995e-07) lv73 = R.call_tir(cls.NT_matmul5, (rms_norm74, model_layers_0_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape144: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv73, R.shape([1, seq_len, 10, 256])) reshape145: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape144, R.shape([seq_len, 10, 256])) lv37 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape145), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape146: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv37, R.shape([1, seq_len, 8, 256])) reshape147: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape146, R.shape([1, seq_len, 2048])) lv74 = R.call_tir(cls.NT_matmul6, (reshape147, model_layers_0_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv72 = R.call_tir(cls.fuse_add_norm_prefill, (lv74, mul38, model_layers_0_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv73_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv72[1] rms_norm75: R.Tensor((1, seq_len, 2048), dtype="float16") = lv72[0] lv75 = R.call_tir(cls.NT_matmul7, (rms_norm75, model_layers_0_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split36: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv75, indices_or_sections=2, axis=-1) split_036: R.Tensor((1, seq_len, 16384), dtype="float16") = split36[0] split_136: R.Tensor((1, seq_len, 16384), dtype="float16") = split36[1] gelu36: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_036) mul39: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu36, split_136) lv76 = R.call_tir(cls.NT_matmul8, (mul39, model_layers_0_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv74_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv76, lv73_1, model_layers_1_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv75_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv74_1[1] rms_norm76: R.Tensor((1, seq_len, 2048), dtype="float16") = lv74_1[0] lv77 = R.call_tir(cls.NT_matmul5, (rms_norm76, model_layers_1_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape148: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv77, R.shape([1, seq_len, 10, 256])) reshape149: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape148, R.shape([seq_len, 10, 256])) lv38 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape149), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape150: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv38, R.shape([1, seq_len, 8, 256])) reshape151: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape150, R.shape([1, seq_len, 2048])) lv78 = R.call_tir(cls.NT_matmul6, (reshape151, model_layers_1_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv76_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv78, lv75_1, model_layers_1_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv77_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv76_1[1] rms_norm77: R.Tensor((1, seq_len, 2048), dtype="float16") = lv76_1[0] lv79 = R.call_tir(cls.NT_matmul7, (rms_norm77, model_layers_1_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split37: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv79, indices_or_sections=2, axis=-1) split_037: R.Tensor((1, seq_len, 16384), dtype="float16") = split37[0] split_137: R.Tensor((1, seq_len, 16384), dtype="float16") = split37[1] gelu37: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_037) mul40: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu37, split_137) lv80 = R.call_tir(cls.NT_matmul8, (mul40, model_layers_1_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv78_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv80, lv77_1, model_layers_2_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv79_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv78_1[1] rms_norm78: R.Tensor((1, seq_len, 2048), dtype="float16") = lv78_1[0] lv81 = R.call_tir(cls.NT_matmul5, (rms_norm78, model_layers_2_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape152: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv81, R.shape([1, seq_len, 10, 256])) reshape153: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape152, R.shape([seq_len, 10, 256])) lv39 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape153), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape154: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv39, R.shape([1, seq_len, 8, 256])) reshape155: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape154, R.shape([1, seq_len, 2048])) lv82 = R.call_tir(cls.NT_matmul6, (reshape155, model_layers_2_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv80_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv82, lv79_1, model_layers_2_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv81_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv80_1[1] rms_norm79: R.Tensor((1, seq_len, 2048), dtype="float16") = lv80_1[0] lv83 = R.call_tir(cls.NT_matmul7, (rms_norm79, model_layers_2_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split38: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv83, indices_or_sections=2, axis=-1) split_038: R.Tensor((1, seq_len, 16384), dtype="float16") = split38[0] split_138: R.Tensor((1, seq_len, 16384), dtype="float16") = split38[1] gelu38: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_038) mul41: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu38, split_138) lv84 = R.call_tir(cls.NT_matmul8, (mul41, model_layers_2_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv82_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv84, lv81_1, model_layers_3_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv83_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv82_1[1] rms_norm80: R.Tensor((1, seq_len, 2048), dtype="float16") = lv82_1[0] lv85 = R.call_tir(cls.NT_matmul5, (rms_norm80, model_layers_3_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape156: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv85, R.shape([1, seq_len, 10, 256])) reshape157: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape156, R.shape([seq_len, 10, 256])) lv40 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape157), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape158: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv40, R.shape([1, seq_len, 8, 256])) reshape159: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape158, R.shape([1, seq_len, 2048])) lv86 = R.call_tir(cls.NT_matmul6, (reshape159, model_layers_3_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv84_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv86, lv83_1, model_layers_3_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv85_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv84_1[1] rms_norm81: R.Tensor((1, seq_len, 2048), dtype="float16") = lv84_1[0] lv87 = R.call_tir(cls.NT_matmul7, (rms_norm81, model_layers_3_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split39: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv87, indices_or_sections=2, axis=-1) split_039: R.Tensor((1, seq_len, 16384), dtype="float16") = split39[0] split_139: R.Tensor((1, seq_len, 16384), dtype="float16") = split39[1] gelu39: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_039) mul42: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu39, split_139) lv88 = R.call_tir(cls.NT_matmul8, (mul42, model_layers_3_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv86_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv88, lv85_1, model_layers_4_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv87_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv86_1[1] rms_norm82: R.Tensor((1, seq_len, 2048), dtype="float16") = lv86_1[0] lv89 = R.call_tir(cls.NT_matmul5, (rms_norm82, model_layers_4_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape160: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv89, R.shape([1, seq_len, 10, 256])) reshape161: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape160, R.shape([seq_len, 10, 256])) lv41 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape161), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape162: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv41, R.shape([1, seq_len, 8, 256])) reshape163: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape162, R.shape([1, seq_len, 2048])) lv90 = R.call_tir(cls.NT_matmul6, (reshape163, model_layers_4_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv88_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv90, lv87_1, model_layers_4_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv89_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv88_1[1] rms_norm83: R.Tensor((1, seq_len, 2048), dtype="float16") = lv88_1[0] lv91 = R.call_tir(cls.NT_matmul7, (rms_norm83, model_layers_4_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split40: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv91, indices_or_sections=2, axis=-1) split_040: R.Tensor((1, seq_len, 16384), dtype="float16") = split40[0] split_140: R.Tensor((1, seq_len, 16384), dtype="float16") = split40[1] gelu40: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_040) mul43: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu40, split_140) lv92 = R.call_tir(cls.NT_matmul8, (mul43, model_layers_4_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv90_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv92, lv89_1, model_layers_5_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv91_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv90_1[1] rms_norm84: R.Tensor((1, seq_len, 2048), dtype="float16") = lv90_1[0] lv93 = R.call_tir(cls.NT_matmul5, (rms_norm84, model_layers_5_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape164: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv93, R.shape([1, seq_len, 10, 256])) reshape165: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape164, R.shape([seq_len, 10, 256])) lv42 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape165), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape166: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv42, R.shape([1, seq_len, 8, 256])) reshape167: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape166, R.shape([1, seq_len, 2048])) lv94 = R.call_tir(cls.NT_matmul6, (reshape167, model_layers_5_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv92_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv94, lv91_1, model_layers_5_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv93_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv92_1[1] rms_norm85: R.Tensor((1, seq_len, 2048), dtype="float16") = lv92_1[0] lv95 = R.call_tir(cls.NT_matmul7, (rms_norm85, model_layers_5_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split41: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv95, indices_or_sections=2, axis=-1) split_041: R.Tensor((1, seq_len, 16384), dtype="float16") = split41[0] split_141: R.Tensor((1, seq_len, 16384), dtype="float16") = split41[1] gelu41: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_041) mul44: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu41, split_141) lv96 = R.call_tir(cls.NT_matmul8, (mul44, model_layers_5_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv94_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv96, lv93_1, model_layers_6_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv95_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv94_1[1] rms_norm86: R.Tensor((1, seq_len, 2048), dtype="float16") = lv94_1[0] lv97 = R.call_tir(cls.NT_matmul5, (rms_norm86, model_layers_6_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape168: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv97, R.shape([1, seq_len, 10, 256])) reshape169: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape168, R.shape([seq_len, 10, 256])) lv43 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape169), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape170: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv43, R.shape([1, seq_len, 8, 256])) reshape171: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape170, R.shape([1, seq_len, 2048])) lv98 = R.call_tir(cls.NT_matmul6, (reshape171, model_layers_6_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv96_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv98, lv95_1, model_layers_6_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv97_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv96_1[1] rms_norm87: R.Tensor((1, seq_len, 2048), dtype="float16") = lv96_1[0] lv99 = R.call_tir(cls.NT_matmul7, (rms_norm87, model_layers_6_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split42: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv99, indices_or_sections=2, axis=-1) split_042: R.Tensor((1, seq_len, 16384), dtype="float16") = split42[0] split_142: R.Tensor((1, seq_len, 16384), dtype="float16") = split42[1] gelu42: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_042) mul45: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu42, split_142) lv100 = R.call_tir(cls.NT_matmul8, (mul45, model_layers_6_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv98_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv100, lv97_1, model_layers_7_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv99_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv98_1[1] rms_norm88: R.Tensor((1, seq_len, 2048), dtype="float16") = lv98_1[0] lv101 = R.call_tir(cls.NT_matmul5, (rms_norm88, model_layers_7_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape172: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv101, R.shape([1, seq_len, 10, 256])) reshape173: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape172, R.shape([seq_len, 10, 256])) lv44 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape173), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape174: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv44, R.shape([1, seq_len, 8, 256])) reshape175: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape174, R.shape([1, seq_len, 2048])) lv102 = R.call_tir(cls.NT_matmul6, (reshape175, model_layers_7_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv100_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv102, lv99_1, model_layers_7_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv101_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv100_1[1] rms_norm89: R.Tensor((1, seq_len, 2048), dtype="float16") = lv100_1[0] lv103 = R.call_tir(cls.NT_matmul7, (rms_norm89, model_layers_7_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split43: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv103, indices_or_sections=2, axis=-1) split_043: R.Tensor((1, seq_len, 16384), dtype="float16") = split43[0] split_143: R.Tensor((1, seq_len, 16384), dtype="float16") = split43[1] gelu43: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_043) mul46: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu43, split_143) lv104 = R.call_tir(cls.NT_matmul8, (mul46, model_layers_7_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv102_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv104, lv101_1, model_layers_8_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv103_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv102_1[1] rms_norm90: R.Tensor((1, seq_len, 2048), dtype="float16") = lv102_1[0] lv105 = R.call_tir(cls.NT_matmul5, (rms_norm90, model_layers_8_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape176: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv105, R.shape([1, seq_len, 10, 256])) reshape177: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape176, R.shape([seq_len, 10, 256])) lv45 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape177), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape178: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv45, R.shape([1, seq_len, 8, 256])) reshape179: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape178, R.shape([1, seq_len, 2048])) lv106 = R.call_tir(cls.NT_matmul6, (reshape179, model_layers_8_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv104_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv106, lv103_1, model_layers_8_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv105_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv104_1[1] rms_norm91: R.Tensor((1, seq_len, 2048), dtype="float16") = lv104_1[0] lv107 = R.call_tir(cls.NT_matmul7, (rms_norm91, model_layers_8_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split44: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv107, indices_or_sections=2, axis=-1) split_044: R.Tensor((1, seq_len, 16384), dtype="float16") = split44[0] split_144: R.Tensor((1, seq_len, 16384), dtype="float16") = split44[1] gelu44: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_044) mul47: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu44, split_144) lv108 = R.call_tir(cls.NT_matmul8, (mul47, model_layers_8_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv106_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv108, lv105_1, model_layers_9_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv107_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv106_1[1] rms_norm92: R.Tensor((1, seq_len, 2048), dtype="float16") = lv106_1[0] lv109 = R.call_tir(cls.NT_matmul5, (rms_norm92, model_layers_9_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape180: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv109, R.shape([1, seq_len, 10, 256])) reshape181: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape180, R.shape([seq_len, 10, 256])) lv46 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape181), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape182: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv46, R.shape([1, seq_len, 8, 256])) reshape183: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape182, R.shape([1, seq_len, 2048])) lv110 = R.call_tir(cls.NT_matmul6, (reshape183, model_layers_9_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv108_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv110, lv107_1, model_layers_9_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv109_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv108_1[1] rms_norm93: R.Tensor((1, seq_len, 2048), dtype="float16") = lv108_1[0] lv111 = R.call_tir(cls.NT_matmul7, (rms_norm93, model_layers_9_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split45: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv111, indices_or_sections=2, axis=-1) split_045: R.Tensor((1, seq_len, 16384), dtype="float16") = split45[0] split_145: R.Tensor((1, seq_len, 16384), dtype="float16") = split45[1] gelu45: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_045) mul48: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu45, split_145) lv112 = R.call_tir(cls.NT_matmul8, (mul48, model_layers_9_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv110_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv112, lv109_1, model_layers_10_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv111_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv110_1[1] rms_norm94: R.Tensor((1, seq_len, 2048), dtype="float16") = lv110_1[0] lv113 = R.call_tir(cls.NT_matmul5, (rms_norm94, model_layers_10_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape184: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv113, R.shape([1, seq_len, 10, 256])) reshape185: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape184, R.shape([seq_len, 10, 256])) lv47 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape185), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape186: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv47, R.shape([1, seq_len, 8, 256])) reshape187: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape186, R.shape([1, seq_len, 2048])) lv114 = R.call_tir(cls.NT_matmul6, (reshape187, model_layers_10_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv112_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv114, lv111_1, model_layers_10_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv113_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv112_1[1] rms_norm95: R.Tensor((1, seq_len, 2048), dtype="float16") = lv112_1[0] lv115 = R.call_tir(cls.NT_matmul7, (rms_norm95, model_layers_10_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split46: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv115, indices_or_sections=2, axis=-1) split_046: R.Tensor((1, seq_len, 16384), dtype="float16") = split46[0] split_146: R.Tensor((1, seq_len, 16384), dtype="float16") = split46[1] gelu46: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_046) mul49: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu46, split_146) lv116 = R.call_tir(cls.NT_matmul8, (mul49, model_layers_10_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv114_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv116, lv113_1, model_layers_11_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv115_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv114_1[1] rms_norm96: R.Tensor((1, seq_len, 2048), dtype="float16") = lv114_1[0] lv117 = R.call_tir(cls.NT_matmul5, (rms_norm96, model_layers_11_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape188: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv117, R.shape([1, seq_len, 10, 256])) reshape189: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape188, R.shape([seq_len, 10, 256])) lv48 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape189), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape190: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv48, R.shape([1, seq_len, 8, 256])) reshape191: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape190, R.shape([1, seq_len, 2048])) lv118 = R.call_tir(cls.NT_matmul6, (reshape191, model_layers_11_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv116_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv118, lv115_1, model_layers_11_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv117_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv116_1[1] rms_norm97: R.Tensor((1, seq_len, 2048), dtype="float16") = lv116_1[0] lv119 = R.call_tir(cls.NT_matmul7, (rms_norm97, model_layers_11_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split47: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv119, indices_or_sections=2, axis=-1) split_047: R.Tensor((1, seq_len, 16384), dtype="float16") = split47[0] split_147: R.Tensor((1, seq_len, 16384), dtype="float16") = split47[1] gelu47: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_047) mul50: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu47, split_147) lv120 = R.call_tir(cls.NT_matmul8, (mul50, model_layers_11_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv118_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv120, lv117_1, model_layers_12_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv119_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv118_1[1] rms_norm98: R.Tensor((1, seq_len, 2048), dtype="float16") = lv118_1[0] lv121 = R.call_tir(cls.NT_matmul5, (rms_norm98, model_layers_12_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape192: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv121, R.shape([1, seq_len, 10, 256])) reshape193: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape192, R.shape([seq_len, 10, 256])) lv49 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape193), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape194: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv49, R.shape([1, seq_len, 8, 256])) reshape195: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape194, R.shape([1, seq_len, 2048])) lv122 = R.call_tir(cls.NT_matmul6, (reshape195, model_layers_12_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv120_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv122, lv119_1, model_layers_12_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv121_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv120_1[1] rms_norm99: R.Tensor((1, seq_len, 2048), dtype="float16") = lv120_1[0] lv123 = R.call_tir(cls.NT_matmul7, (rms_norm99, model_layers_12_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split48: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv123, indices_or_sections=2, axis=-1) split_048: R.Tensor((1, seq_len, 16384), dtype="float16") = split48[0] split_148: R.Tensor((1, seq_len, 16384), dtype="float16") = split48[1] gelu48: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_048) mul51: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu48, split_148) lv124 = R.call_tir(cls.NT_matmul8, (mul51, model_layers_12_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv122_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv124, lv121_1, model_layers_13_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv123_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv122_1[1] rms_norm100: R.Tensor((1, seq_len, 2048), dtype="float16") = lv122_1[0] lv125 = R.call_tir(cls.NT_matmul5, (rms_norm100, model_layers_13_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape196: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv125, R.shape([1, seq_len, 10, 256])) reshape197: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape196, R.shape([seq_len, 10, 256])) lv50 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape197), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape198: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv50, R.shape([1, seq_len, 8, 256])) reshape199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape198, R.shape([1, seq_len, 2048])) lv126 = R.call_tir(cls.NT_matmul6, (reshape199, model_layers_13_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv124_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv126, lv123_1, model_layers_13_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv125_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv124_1[1] rms_norm101: R.Tensor((1, seq_len, 2048), dtype="float16") = lv124_1[0] lv127 = R.call_tir(cls.NT_matmul7, (rms_norm101, model_layers_13_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split49: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv127, indices_or_sections=2, axis=-1) split_049: R.Tensor((1, seq_len, 16384), dtype="float16") = split49[0] split_149: R.Tensor((1, seq_len, 16384), dtype="float16") = split49[1] gelu49: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_049) mul52: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu49, split_149) lv128 = R.call_tir(cls.NT_matmul8, (mul52, model_layers_13_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv126_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv128, lv125_1, model_layers_14_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv127_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv126_1[1] rms_norm102: R.Tensor((1, seq_len, 2048), dtype="float16") = lv126_1[0] lv129 = R.call_tir(cls.NT_matmul5, (rms_norm102, model_layers_14_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape200: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv129, R.shape([1, seq_len, 10, 256])) reshape201: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape200, R.shape([seq_len, 10, 256])) lv51 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape201), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape202: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv51, R.shape([1, seq_len, 8, 256])) reshape203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape202, R.shape([1, seq_len, 2048])) lv130 = R.call_tir(cls.NT_matmul6, (reshape203, model_layers_14_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv128_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv130, lv127_1, model_layers_14_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv129_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv128_1[1] rms_norm103: R.Tensor((1, seq_len, 2048), dtype="float16") = lv128_1[0] lv131 = R.call_tir(cls.NT_matmul7, (rms_norm103, model_layers_14_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split50: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv131, indices_or_sections=2, axis=-1) split_050: R.Tensor((1, seq_len, 16384), dtype="float16") = split50[0] split_150: R.Tensor((1, seq_len, 16384), dtype="float16") = split50[1] gelu50: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_050) mul53: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu50, split_150) lv132 = R.call_tir(cls.NT_matmul8, (mul53, model_layers_14_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv130_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv132, lv129_1, model_layers_15_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv131_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv130_1[1] rms_norm104: R.Tensor((1, seq_len, 2048), dtype="float16") = lv130_1[0] lv133 = R.call_tir(cls.NT_matmul5, (rms_norm104, model_layers_15_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape204: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv133, R.shape([1, seq_len, 10, 256])) reshape205: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape204, R.shape([seq_len, 10, 256])) lv52 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape205), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape206: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv52, R.shape([1, seq_len, 8, 256])) reshape207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape206, R.shape([1, seq_len, 2048])) lv134 = R.call_tir(cls.NT_matmul6, (reshape207, model_layers_15_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv132_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv134, lv131_1, model_layers_15_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv133_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv132_1[1] rms_norm105: R.Tensor((1, seq_len, 2048), dtype="float16") = lv132_1[0] lv135 = R.call_tir(cls.NT_matmul7, (rms_norm105, model_layers_15_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split51: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv135, indices_or_sections=2, axis=-1) split_051: R.Tensor((1, seq_len, 16384), dtype="float16") = split51[0] split_151: R.Tensor((1, seq_len, 16384), dtype="float16") = split51[1] gelu51: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_051) mul54: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu51, split_151) lv136 = R.call_tir(cls.NT_matmul8, (mul54, model_layers_15_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv134_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv136, lv133_1, model_layers_16_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv135_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv134_1[1] rms_norm106: R.Tensor((1, seq_len, 2048), dtype="float16") = lv134_1[0] lv137 = R.call_tir(cls.NT_matmul5, (rms_norm106, model_layers_16_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape208: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv137, R.shape([1, seq_len, 10, 256])) reshape209: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape208, R.shape([seq_len, 10, 256])) lv53 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape209), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape210: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv53, R.shape([1, seq_len, 8, 256])) reshape211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape210, R.shape([1, seq_len, 2048])) lv138 = R.call_tir(cls.NT_matmul6, (reshape211, model_layers_16_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv136_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv138, lv135_1, model_layers_16_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv137_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv136_1[1] rms_norm107: R.Tensor((1, seq_len, 2048), dtype="float16") = lv136_1[0] lv139 = R.call_tir(cls.NT_matmul7, (rms_norm107, model_layers_16_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split52: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv139, indices_or_sections=2, axis=-1) split_052: R.Tensor((1, seq_len, 16384), dtype="float16") = split52[0] split_152: R.Tensor((1, seq_len, 16384), dtype="float16") = split52[1] gelu52: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_052) mul55: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu52, split_152) lv140 = R.call_tir(cls.NT_matmul8, (mul55, model_layers_16_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv138_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv140, lv137_1, model_layers_17_input_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv139_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv138_1[1] rms_norm108: R.Tensor((1, seq_len, 2048), dtype="float16") = lv138_1[0] lv141 = R.call_tir(cls.NT_matmul5, (rms_norm108, model_layers_17_self_attn_qkv_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape212: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv141, R.shape([1, seq_len, 10, 256])) reshape213: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape212, R.shape([seq_len, 10, 256])) lv54 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape213), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape214: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv54, R.shape([1, seq_len, 8, 256])) reshape215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape214, R.shape([1, seq_len, 2048])) lv142 = R.call_tir(cls.NT_matmul6, (reshape215, model_layers_17_self_attn_o_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv140_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv142, lv139_1, model_layers_17_post_attention_layernorm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv141_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv140_1[1] rms_norm109: R.Tensor((1, seq_len, 2048), dtype="float16") = lv140_1[0] lv143 = R.call_tir(cls.NT_matmul7, (rms_norm109, model_layers_17_mlp_gate_up_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split53: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv143, indices_or_sections=2, axis=-1) split_053: R.Tensor((1, seq_len, 16384), dtype="float16") = split53[0] split_153: R.Tensor((1, seq_len, 16384), dtype="float16") = split53[1] gelu53: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_053) mul56: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu53, split_153) lv144 = R.call_tir(cls.NT_matmul8, (mul56, model_layers_17_mlp_down_proj_weight3), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv142_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv144, lv141_1, model_norm_weight3), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) rms_norm110: R.Tensor((1, seq_len, 2048), dtype="float16") = lv142_1[0] take1: R.Tensor((1, batch_size, 2048), dtype="float16") = R.take(rms_norm110, logit_positions, axis=1) lv145 = R.call_tir(cls.NT_matmul9, (take1, model_embed_tokens_weight3), out_sinfo=R.Tensor((1, batch_size, vocab_size), dtype="float32")) gv3: R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype="float32"), R.Object) = lv145, paged_kv_cache R.output(gv3) return gv3 @R.function def batch_verify(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): model_embed_tokens_weight5: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[109] mul76: R.Tensor((1, seq_len, 2048), dtype="float16") = R.multiply(input_embeds, R.const(45.25, "float16")) rms_norm148: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(mul76, model_layers_0_input_layernorm_weight5, axes=[-1], epsilon=9.9999999999999995e-07) lv146 = R.call_tir(cls.NT_matmul5, (rms_norm148, model_layers_0_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape288: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv146, R.shape([1, seq_len, 10, 256])) reshape289: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape288, R.shape([seq_len, 10, 256])) lv73 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape289), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape290: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv73, R.shape([1, seq_len, 8, 256])) reshape291: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape290, R.shape([1, seq_len, 2048])) lv147 = R.call_tir(cls.NT_matmul6, (reshape291, model_layers_0_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv144 = R.call_tir(cls.fuse_add_norm_prefill, (lv147, mul76, model_layers_0_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv145: R.Tensor((1, seq_len, 2048), dtype="float16") = lv144[1] rms_norm149: R.Tensor((1, seq_len, 2048), dtype="float16") = lv144[0] lv148 = R.call_tir(cls.NT_matmul7, (rms_norm149, model_layers_0_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split72: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv148, indices_or_sections=2, axis=-1) split_072: R.Tensor((1, seq_len, 16384), dtype="float16") = split72[0] split_172: R.Tensor((1, seq_len, 16384), dtype="float16") = split72[1] gelu72: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_072) mul77: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu72, split_172) lv149 = R.call_tir(cls.NT_matmul8, (mul77, model_layers_0_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv146_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv149, lv145, model_layers_1_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv147_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv146_1[1] rms_norm150: R.Tensor((1, seq_len, 2048), dtype="float16") = lv146_1[0] lv150 = R.call_tir(cls.NT_matmul5, (rms_norm150, model_layers_1_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape292: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv150, R.shape([1, seq_len, 10, 256])) reshape293: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape292, R.shape([seq_len, 10, 256])) lv74 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape293), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape294: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv74, R.shape([1, seq_len, 8, 256])) reshape295: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape294, R.shape([1, seq_len, 2048])) lv151 = R.call_tir(cls.NT_matmul6, (reshape295, model_layers_1_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv148_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv151, lv147_1, model_layers_1_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv149_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv148_1[1] rms_norm151: R.Tensor((1, seq_len, 2048), dtype="float16") = lv148_1[0] lv152 = R.call_tir(cls.NT_matmul7, (rms_norm151, model_layers_1_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split73: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv152, indices_or_sections=2, axis=-1) split_073: R.Tensor((1, seq_len, 16384), dtype="float16") = split73[0] split_173: R.Tensor((1, seq_len, 16384), dtype="float16") = split73[1] gelu73: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_073) mul78: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu73, split_173) lv153 = R.call_tir(cls.NT_matmul8, (mul78, model_layers_1_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv150_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv153, lv149_1, model_layers_2_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv151_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv150_1[1] rms_norm152: R.Tensor((1, seq_len, 2048), dtype="float16") = lv150_1[0] lv154 = R.call_tir(cls.NT_matmul5, (rms_norm152, model_layers_2_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape296: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv154, R.shape([1, seq_len, 10, 256])) reshape297: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape296, R.shape([seq_len, 10, 256])) lv75 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape297), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape298: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv75, R.shape([1, seq_len, 8, 256])) reshape299: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape298, R.shape([1, seq_len, 2048])) lv155 = R.call_tir(cls.NT_matmul6, (reshape299, model_layers_2_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv152_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv155, lv151_1, model_layers_2_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv153_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv152_1[1] rms_norm153: R.Tensor((1, seq_len, 2048), dtype="float16") = lv152_1[0] lv156 = R.call_tir(cls.NT_matmul7, (rms_norm153, model_layers_2_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split74: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv156, indices_or_sections=2, axis=-1) split_074: R.Tensor((1, seq_len, 16384), dtype="float16") = split74[0] split_174: R.Tensor((1, seq_len, 16384), dtype="float16") = split74[1] gelu74: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_074) mul79: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu74, split_174) lv157 = R.call_tir(cls.NT_matmul8, (mul79, model_layers_2_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv154_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv157, lv153_1, model_layers_3_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv155_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv154_1[1] rms_norm154: R.Tensor((1, seq_len, 2048), dtype="float16") = lv154_1[0] lv158 = R.call_tir(cls.NT_matmul5, (rms_norm154, model_layers_3_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape300: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv158, R.shape([1, seq_len, 10, 256])) reshape301: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape300, R.shape([seq_len, 10, 256])) lv76 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape301), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape302: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv76, R.shape([1, seq_len, 8, 256])) reshape303: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape302, R.shape([1, seq_len, 2048])) lv159 = R.call_tir(cls.NT_matmul6, (reshape303, model_layers_3_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv156_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv159, lv155_1, model_layers_3_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv157_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv156_1[1] rms_norm155: R.Tensor((1, seq_len, 2048), dtype="float16") = lv156_1[0] lv160 = R.call_tir(cls.NT_matmul7, (rms_norm155, model_layers_3_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split75: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv160, indices_or_sections=2, axis=-1) split_075: R.Tensor((1, seq_len, 16384), dtype="float16") = split75[0] split_175: R.Tensor((1, seq_len, 16384), dtype="float16") = split75[1] gelu75: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_075) mul80: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu75, split_175) lv161 = R.call_tir(cls.NT_matmul8, (mul80, model_layers_3_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv158_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv161, lv157_1, model_layers_4_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv159_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv158_1[1] rms_norm156: R.Tensor((1, seq_len, 2048), dtype="float16") = lv158_1[0] lv162 = R.call_tir(cls.NT_matmul5, (rms_norm156, model_layers_4_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape304: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv162, R.shape([1, seq_len, 10, 256])) reshape305: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape304, R.shape([seq_len, 10, 256])) lv77 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape305), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape306: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv77, R.shape([1, seq_len, 8, 256])) reshape307: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape306, R.shape([1, seq_len, 2048])) lv163 = R.call_tir(cls.NT_matmul6, (reshape307, model_layers_4_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv160_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv163, lv159_1, model_layers_4_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv161_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv160_1[1] rms_norm157: R.Tensor((1, seq_len, 2048), dtype="float16") = lv160_1[0] lv164 = R.call_tir(cls.NT_matmul7, (rms_norm157, model_layers_4_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split76: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv164, indices_or_sections=2, axis=-1) split_076: R.Tensor((1, seq_len, 16384), dtype="float16") = split76[0] split_176: R.Tensor((1, seq_len, 16384), dtype="float16") = split76[1] gelu76: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_076) mul81: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu76, split_176) lv165 = R.call_tir(cls.NT_matmul8, (mul81, model_layers_4_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv162_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv165, lv161_1, model_layers_5_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv163_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv162_1[1] rms_norm158: R.Tensor((1, seq_len, 2048), dtype="float16") = lv162_1[0] lv166 = R.call_tir(cls.NT_matmul5, (rms_norm158, model_layers_5_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape308: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv166, R.shape([1, seq_len, 10, 256])) reshape309: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape308, R.shape([seq_len, 10, 256])) lv78 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape309), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape310: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv78, R.shape([1, seq_len, 8, 256])) reshape311: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape310, R.shape([1, seq_len, 2048])) lv167 = R.call_tir(cls.NT_matmul6, (reshape311, model_layers_5_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv164_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv167, lv163_1, model_layers_5_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv165_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv164_1[1] rms_norm159: R.Tensor((1, seq_len, 2048), dtype="float16") = lv164_1[0] lv168 = R.call_tir(cls.NT_matmul7, (rms_norm159, model_layers_5_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split77: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv168, indices_or_sections=2, axis=-1) split_077: R.Tensor((1, seq_len, 16384), dtype="float16") = split77[0] split_177: R.Tensor((1, seq_len, 16384), dtype="float16") = split77[1] gelu77: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_077) mul82: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu77, split_177) lv169 = R.call_tir(cls.NT_matmul8, (mul82, model_layers_5_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv166_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv169, lv165_1, model_layers_6_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv167_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv166_1[1] rms_norm160: R.Tensor((1, seq_len, 2048), dtype="float16") = lv166_1[0] lv170 = R.call_tir(cls.NT_matmul5, (rms_norm160, model_layers_6_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape312: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv170, R.shape([1, seq_len, 10, 256])) reshape313: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape312, R.shape([seq_len, 10, 256])) lv79 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape313), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape314: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv79, R.shape([1, seq_len, 8, 256])) reshape315: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape314, R.shape([1, seq_len, 2048])) lv171 = R.call_tir(cls.NT_matmul6, (reshape315, model_layers_6_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv168_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv171, lv167_1, model_layers_6_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv169_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv168_1[1] rms_norm161: R.Tensor((1, seq_len, 2048), dtype="float16") = lv168_1[0] lv172 = R.call_tir(cls.NT_matmul7, (rms_norm161, model_layers_6_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split78: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv172, indices_or_sections=2, axis=-1) split_078: R.Tensor((1, seq_len, 16384), dtype="float16") = split78[0] split_178: R.Tensor((1, seq_len, 16384), dtype="float16") = split78[1] gelu78: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_078) mul83: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu78, split_178) lv173 = R.call_tir(cls.NT_matmul8, (mul83, model_layers_6_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv170_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv173, lv169_1, model_layers_7_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv171_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv170_1[1] rms_norm162: R.Tensor((1, seq_len, 2048), dtype="float16") = lv170_1[0] lv174 = R.call_tir(cls.NT_matmul5, (rms_norm162, model_layers_7_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape316: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv174, R.shape([1, seq_len, 10, 256])) reshape317: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape316, R.shape([seq_len, 10, 256])) lv80 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape317), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape318: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv80, R.shape([1, seq_len, 8, 256])) reshape319: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape318, R.shape([1, seq_len, 2048])) lv175 = R.call_tir(cls.NT_matmul6, (reshape319, model_layers_7_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv172_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv175, lv171_1, model_layers_7_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv173_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv172_1[1] rms_norm163: R.Tensor((1, seq_len, 2048), dtype="float16") = lv172_1[0] lv176 = R.call_tir(cls.NT_matmul7, (rms_norm163, model_layers_7_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split79: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv176, indices_or_sections=2, axis=-1) split_079: R.Tensor((1, seq_len, 16384), dtype="float16") = split79[0] split_179: R.Tensor((1, seq_len, 16384), dtype="float16") = split79[1] gelu79: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_079) mul84: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu79, split_179) lv177 = R.call_tir(cls.NT_matmul8, (mul84, model_layers_7_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv174_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv177, lv173_1, model_layers_8_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv175_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv174_1[1] rms_norm164: R.Tensor((1, seq_len, 2048), dtype="float16") = lv174_1[0] lv178 = R.call_tir(cls.NT_matmul5, (rms_norm164, model_layers_8_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape320: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv178, R.shape([1, seq_len, 10, 256])) reshape321: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape320, R.shape([seq_len, 10, 256])) lv81 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape321), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape322: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv81, R.shape([1, seq_len, 8, 256])) reshape323: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape322, R.shape([1, seq_len, 2048])) lv179 = R.call_tir(cls.NT_matmul6, (reshape323, model_layers_8_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv176_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv179, lv175_1, model_layers_8_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv177_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv176_1[1] rms_norm165: R.Tensor((1, seq_len, 2048), dtype="float16") = lv176_1[0] lv180 = R.call_tir(cls.NT_matmul7, (rms_norm165, model_layers_8_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split80: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv180, indices_or_sections=2, axis=-1) split_080: R.Tensor((1, seq_len, 16384), dtype="float16") = split80[0] split_180: R.Tensor((1, seq_len, 16384), dtype="float16") = split80[1] gelu80: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_080) mul85: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu80, split_180) lv181 = R.call_tir(cls.NT_matmul8, (mul85, model_layers_8_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv178_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv181, lv177_1, model_layers_9_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv179_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv178_1[1] rms_norm166: R.Tensor((1, seq_len, 2048), dtype="float16") = lv178_1[0] lv182 = R.call_tir(cls.NT_matmul5, (rms_norm166, model_layers_9_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape324: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv182, R.shape([1, seq_len, 10, 256])) reshape325: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape324, R.shape([seq_len, 10, 256])) lv82 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape325), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape326: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv82, R.shape([1, seq_len, 8, 256])) reshape327: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape326, R.shape([1, seq_len, 2048])) lv183 = R.call_tir(cls.NT_matmul6, (reshape327, model_layers_9_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv180_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv183, lv179_1, model_layers_9_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv181_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv180_1[1] rms_norm167: R.Tensor((1, seq_len, 2048), dtype="float16") = lv180_1[0] lv184 = R.call_tir(cls.NT_matmul7, (rms_norm167, model_layers_9_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split81: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv184, indices_or_sections=2, axis=-1) split_081: R.Tensor((1, seq_len, 16384), dtype="float16") = split81[0] split_181: R.Tensor((1, seq_len, 16384), dtype="float16") = split81[1] gelu81: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_081) mul86: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu81, split_181) lv185 = R.call_tir(cls.NT_matmul8, (mul86, model_layers_9_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv182_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv185, lv181_1, model_layers_10_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv183_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv182_1[1] rms_norm168: R.Tensor((1, seq_len, 2048), dtype="float16") = lv182_1[0] lv186 = R.call_tir(cls.NT_matmul5, (rms_norm168, model_layers_10_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape328: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv186, R.shape([1, seq_len, 10, 256])) reshape329: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape328, R.shape([seq_len, 10, 256])) lv83 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape329), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape330: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv83, R.shape([1, seq_len, 8, 256])) reshape331: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape330, R.shape([1, seq_len, 2048])) lv187 = R.call_tir(cls.NT_matmul6, (reshape331, model_layers_10_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv184_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv187, lv183_1, model_layers_10_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv185_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv184_1[1] rms_norm169: R.Tensor((1, seq_len, 2048), dtype="float16") = lv184_1[0] lv188 = R.call_tir(cls.NT_matmul7, (rms_norm169, model_layers_10_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split82: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv188, indices_or_sections=2, axis=-1) split_082: R.Tensor((1, seq_len, 16384), dtype="float16") = split82[0] split_182: R.Tensor((1, seq_len, 16384), dtype="float16") = split82[1] gelu82: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_082) mul87: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu82, split_182) lv189 = R.call_tir(cls.NT_matmul8, (mul87, model_layers_10_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv186_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv189, lv185_1, model_layers_11_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv187_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv186_1[1] rms_norm170: R.Tensor((1, seq_len, 2048), dtype="float16") = lv186_1[0] lv190 = R.call_tir(cls.NT_matmul5, (rms_norm170, model_layers_11_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape332: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv190, R.shape([1, seq_len, 10, 256])) reshape333: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape332, R.shape([seq_len, 10, 256])) lv84 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape333), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape334: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv84, R.shape([1, seq_len, 8, 256])) reshape335: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape334, R.shape([1, seq_len, 2048])) lv191 = R.call_tir(cls.NT_matmul6, (reshape335, model_layers_11_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv188_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv191, lv187_1, model_layers_11_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv189_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv188_1[1] rms_norm171: R.Tensor((1, seq_len, 2048), dtype="float16") = lv188_1[0] lv192 = R.call_tir(cls.NT_matmul7, (rms_norm171, model_layers_11_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split83: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv192, indices_or_sections=2, axis=-1) split_083: R.Tensor((1, seq_len, 16384), dtype="float16") = split83[0] split_183: R.Tensor((1, seq_len, 16384), dtype="float16") = split83[1] gelu83: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_083) mul88: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu83, split_183) lv193 = R.call_tir(cls.NT_matmul8, (mul88, model_layers_11_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv190_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv193, lv189_1, model_layers_12_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv191_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv190_1[1] rms_norm172: R.Tensor((1, seq_len, 2048), dtype="float16") = lv190_1[0] lv194 = R.call_tir(cls.NT_matmul5, (rms_norm172, model_layers_12_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape336: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv194, R.shape([1, seq_len, 10, 256])) reshape337: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape336, R.shape([seq_len, 10, 256])) lv85 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape337), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape338: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv85, R.shape([1, seq_len, 8, 256])) reshape339: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape338, R.shape([1, seq_len, 2048])) lv195 = R.call_tir(cls.NT_matmul6, (reshape339, model_layers_12_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv192_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv195, lv191_1, model_layers_12_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv193_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv192_1[1] rms_norm173: R.Tensor((1, seq_len, 2048), dtype="float16") = lv192_1[0] lv196 = R.call_tir(cls.NT_matmul7, (rms_norm173, model_layers_12_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split84: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv196, indices_or_sections=2, axis=-1) split_084: R.Tensor((1, seq_len, 16384), dtype="float16") = split84[0] split_184: R.Tensor((1, seq_len, 16384), dtype="float16") = split84[1] gelu84: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_084) mul89: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu84, split_184) lv197 = R.call_tir(cls.NT_matmul8, (mul89, model_layers_12_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv194_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv197, lv193_1, model_layers_13_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv195_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv194_1[1] rms_norm174: R.Tensor((1, seq_len, 2048), dtype="float16") = lv194_1[0] lv198 = R.call_tir(cls.NT_matmul5, (rms_norm174, model_layers_13_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape340: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv198, R.shape([1, seq_len, 10, 256])) reshape341: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape340, R.shape([seq_len, 10, 256])) lv86 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape341), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape342: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv86, R.shape([1, seq_len, 8, 256])) reshape343: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape342, R.shape([1, seq_len, 2048])) lv199 = R.call_tir(cls.NT_matmul6, (reshape343, model_layers_13_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv196_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv199, lv195_1, model_layers_13_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv197_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv196_1[1] rms_norm175: R.Tensor((1, seq_len, 2048), dtype="float16") = lv196_1[0] lv200 = R.call_tir(cls.NT_matmul7, (rms_norm175, model_layers_13_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split85: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv200, indices_or_sections=2, axis=-1) split_085: R.Tensor((1, seq_len, 16384), dtype="float16") = split85[0] split_185: R.Tensor((1, seq_len, 16384), dtype="float16") = split85[1] gelu85: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_085) mul90: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu85, split_185) lv201 = R.call_tir(cls.NT_matmul8, (mul90, model_layers_13_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv198_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv201, lv197_1, model_layers_14_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv199_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv198_1[1] rms_norm176: R.Tensor((1, seq_len, 2048), dtype="float16") = lv198_1[0] lv202 = R.call_tir(cls.NT_matmul5, (rms_norm176, model_layers_14_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape344: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv202, R.shape([1, seq_len, 10, 256])) reshape345: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape344, R.shape([seq_len, 10, 256])) lv87 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape345), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape346: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv87, R.shape([1, seq_len, 8, 256])) reshape347: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape346, R.shape([1, seq_len, 2048])) lv203 = R.call_tir(cls.NT_matmul6, (reshape347, model_layers_14_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv200_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv203, lv199_1, model_layers_14_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv201_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv200_1[1] rms_norm177: R.Tensor((1, seq_len, 2048), dtype="float16") = lv200_1[0] lv204 = R.call_tir(cls.NT_matmul7, (rms_norm177, model_layers_14_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split86: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv204, indices_or_sections=2, axis=-1) split_086: R.Tensor((1, seq_len, 16384), dtype="float16") = split86[0] split_186: R.Tensor((1, seq_len, 16384), dtype="float16") = split86[1] gelu86: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_086) mul91: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu86, split_186) lv205 = R.call_tir(cls.NT_matmul8, (mul91, model_layers_14_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv202_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv205, lv201_1, model_layers_15_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv203_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv202_1[1] rms_norm178: R.Tensor((1, seq_len, 2048), dtype="float16") = lv202_1[0] lv206 = R.call_tir(cls.NT_matmul5, (rms_norm178, model_layers_15_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape348: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv206, R.shape([1, seq_len, 10, 256])) reshape349: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape348, R.shape([seq_len, 10, 256])) lv88 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape349), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape350: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv88, R.shape([1, seq_len, 8, 256])) reshape351: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape350, R.shape([1, seq_len, 2048])) lv207 = R.call_tir(cls.NT_matmul6, (reshape351, model_layers_15_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv204_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv207, lv203_1, model_layers_15_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv205_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv204_1[1] rms_norm179: R.Tensor((1, seq_len, 2048), dtype="float16") = lv204_1[0] lv208 = R.call_tir(cls.NT_matmul7, (rms_norm179, model_layers_15_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split87: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv208, indices_or_sections=2, axis=-1) split_087: R.Tensor((1, seq_len, 16384), dtype="float16") = split87[0] split_187: R.Tensor((1, seq_len, 16384), dtype="float16") = split87[1] gelu87: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_087) mul92: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu87, split_187) lv209 = R.call_tir(cls.NT_matmul8, (mul92, model_layers_15_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv206_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv209, lv205_1, model_layers_16_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv207_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv206_1[1] rms_norm180: R.Tensor((1, seq_len, 2048), dtype="float16") = lv206_1[0] lv210 = R.call_tir(cls.NT_matmul5, (rms_norm180, model_layers_16_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape352: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv210, R.shape([1, seq_len, 10, 256])) reshape353: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape352, R.shape([seq_len, 10, 256])) lv89 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape353), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape354: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv89, R.shape([1, seq_len, 8, 256])) reshape355: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape354, R.shape([1, seq_len, 2048])) lv211 = R.call_tir(cls.NT_matmul6, (reshape355, model_layers_16_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv208_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv211, lv207_1, model_layers_16_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv209_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv208_1[1] rms_norm181: R.Tensor((1, seq_len, 2048), dtype="float16") = lv208_1[0] lv212 = R.call_tir(cls.NT_matmul7, (rms_norm181, model_layers_16_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split88: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv212, indices_or_sections=2, axis=-1) split_088: R.Tensor((1, seq_len, 16384), dtype="float16") = split88[0] split_188: R.Tensor((1, seq_len, 16384), dtype="float16") = split88[1] gelu88: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_088) mul93: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu88, split_188) lv213 = R.call_tir(cls.NT_matmul8, (mul93, model_layers_16_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv210_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv213, lv209_1, model_layers_17_input_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv211_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv210_1[1] rms_norm182: R.Tensor((1, seq_len, 2048), dtype="float16") = lv210_1[0] lv214 = R.call_tir(cls.NT_matmul5, (rms_norm182, model_layers_17_self_attn_qkv_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape356: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv214, R.shape([1, seq_len, 10, 256])) reshape357: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape356, R.shape([seq_len, 10, 256])) lv90 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape357), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape358: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv90, R.shape([1, seq_len, 8, 256])) reshape359: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape358, R.shape([1, seq_len, 2048])) lv215 = R.call_tir(cls.NT_matmul6, (reshape359, model_layers_17_self_attn_o_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv212_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv215, lv211_1, model_layers_17_post_attention_layernorm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv213_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv212_1[1] rms_norm183: R.Tensor((1, seq_len, 2048), dtype="float16") = lv212_1[0] lv216 = R.call_tir(cls.NT_matmul7, (rms_norm183, model_layers_17_mlp_gate_up_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split89: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv216, indices_or_sections=2, axis=-1) split_089: R.Tensor((1, seq_len, 16384), dtype="float16") = split89[0] split_189: R.Tensor((1, seq_len, 16384), dtype="float16") = split89[1] gelu89: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_089) mul94: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu89, split_189) lv217 = R.call_tir(cls.NT_matmul8, (mul94, model_layers_17_mlp_down_proj_weight5), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv214_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv217, lv213_1, model_norm_weight5), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) rms_norm184: R.Tensor((1, seq_len, 2048), dtype="float16") = lv214_1[0] lv218 = R.call_tir(cls.NT_matmul9, (rms_norm184, model_embed_tokens_weight5), out_sinfo=R.Tensor((1, seq_len, vocab_size), dtype="float32")) gv5: R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype="float32"), R.Object) = lv218, paged_kv_cache R.output(gv5) return gv5 @R.function def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: max_batch_size = T.int64() max_total_seq_len = T.int64() prefill_chunk_size = T.int64() page_size = T.int64() support_sliding_window = T.int64() R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module gv: R.Tensor((), dtype="float16") = R.zeros(R.shape([]), dtype="float16") paged_kv_cache: R.Object = R.call_pure_packed("vm.builtin.paged_attention_kv_cache_create_reduced", R.shape([max_batch_size, max_total_seq_len, prefill_chunk_size, page_size, support_sliding_window]), R.shape([0, 18]), R.prim_value(8), R.prim_value(1), R.prim_value(256), R.prim_value(1), R.prim_value(1), R.prim_value(T.float32(10000.0)), gv, cls.tir_kv_cache_transpose_append, cls.batch_prefill_paged_kv, cls.batch_decode_paged_kv, cls.batch_prefill_paged_kv_sliding_window, cls.batch_decode_paged_kv_sliding_window, cls.batch_prefill_ragged_kv, cls.merge_state_inplace, cls.fused_rope, cls.copy_single_page, cls.tir_kv_cache_debug_get_kv, cls.compact_kv_copy, cls.batch_tree_attn, cls.tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) return paged_kv_cache @R.function def decode(input_embed: R.Tensor((1, 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): model_embed_tokens_weight2: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[109] mul19: R.Tensor((1, 1, 2048), dtype="float16") = R.multiply(input_embed, R.const(45.25, "float16")) rms_norm37: R.Tensor((1, 1, 2048), dtype="float16") = R.nn.rms_norm(mul19, model_layers_0_input_layernorm_weight2, axes=[-1], epsilon=9.9999999999999995e-07) lv219 = R.call_tir(cls.NT_matmul10, (rms_norm37, model_layers_0_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape72: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv219, R.shape([1, 1, 10, 256])) reshape73: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape72, R.shape([1, 10, 256])) lv19 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape73), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape74: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv19, R.shape([1, 1, 8, 256])) reshape75: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape74, R.shape([1, 1, 2048])) lv220 = R.call_tir(cls.NT_matmul11, (reshape75, model_layers_0_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv216 = R.call_tir(cls.fuse_add_norm_prefill, (lv220, mul19, model_layers_0_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv217: R.Tensor((1, 1, 2048), dtype="float16") = lv216[1] rms_norm38: R.Tensor((1, 1, 2048), dtype="float16") = lv216[0] lv221 = R.call_tir(cls.NT_matmul12, (rms_norm38, model_layers_0_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split18: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv221, indices_or_sections=2, axis=-1) split_018: R.Tensor((1, 1, 16384), dtype="float16") = split18[0] split_118: R.Tensor((1, 1, 16384), dtype="float16") = split18[1] gelu18: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_018) mul20: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu18, split_118) lv222 = R.call_tir(cls.NT_matmul13, (mul20, model_layers_0_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv218 = R.call_tir(cls.fuse_add_norm_prefill, (lv222, lv217, model_layers_1_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv219_1: R.Tensor((1, 1, 2048), dtype="float16") = lv218[1] rms_norm39: R.Tensor((1, 1, 2048), dtype="float16") = lv218[0] lv223 = R.call_tir(cls.NT_matmul10, (rms_norm39, model_layers_1_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape76: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv223, R.shape([1, 1, 10, 256])) reshape77: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape76, R.shape([1, 10, 256])) lv20 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape77), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape78: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv20, R.shape([1, 1, 8, 256])) reshape79: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape78, R.shape([1, 1, 2048])) lv224 = R.call_tir(cls.NT_matmul11, (reshape79, model_layers_1_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv220_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv224, lv219_1, model_layers_1_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv221_1: R.Tensor((1, 1, 2048), dtype="float16") = lv220_1[1] rms_norm40: R.Tensor((1, 1, 2048), dtype="float16") = lv220_1[0] lv225 = R.call_tir(cls.NT_matmul12, (rms_norm40, model_layers_1_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split19: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv225, indices_or_sections=2, axis=-1) split_019: R.Tensor((1, 1, 16384), dtype="float16") = split19[0] split_119: R.Tensor((1, 1, 16384), dtype="float16") = split19[1] gelu19: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_019) mul21: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu19, split_119) lv226 = R.call_tir(cls.NT_matmul13, (mul21, model_layers_1_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv222_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv226, lv221_1, model_layers_2_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv223_1: R.Tensor((1, 1, 2048), dtype="float16") = lv222_1[1] rms_norm41: R.Tensor((1, 1, 2048), dtype="float16") = lv222_1[0] lv227 = R.call_tir(cls.NT_matmul10, (rms_norm41, model_layers_2_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape80: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv227, R.shape([1, 1, 10, 256])) reshape81: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape80, R.shape([1, 10, 256])) lv21 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape81), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape82: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv21, R.shape([1, 1, 8, 256])) reshape83: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape82, R.shape([1, 1, 2048])) lv228 = R.call_tir(cls.NT_matmul11, (reshape83, model_layers_2_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv224_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv228, lv223_1, model_layers_2_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv225_1: R.Tensor((1, 1, 2048), dtype="float16") = lv224_1[1] rms_norm42: R.Tensor((1, 1, 2048), dtype="float16") = lv224_1[0] lv229 = R.call_tir(cls.NT_matmul12, (rms_norm42, model_layers_2_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split20: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv229, indices_or_sections=2, axis=-1) split_020: R.Tensor((1, 1, 16384), dtype="float16") = split20[0] split_120: R.Tensor((1, 1, 16384), dtype="float16") = split20[1] gelu20: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_020) mul22: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu20, split_120) lv230 = R.call_tir(cls.NT_matmul13, (mul22, model_layers_2_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv226_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv230, lv225_1, model_layers_3_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv227_1: R.Tensor((1, 1, 2048), dtype="float16") = lv226_1[1] rms_norm43: R.Tensor((1, 1, 2048), dtype="float16") = lv226_1[0] lv231 = R.call_tir(cls.NT_matmul10, (rms_norm43, model_layers_3_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape84: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv231, R.shape([1, 1, 10, 256])) reshape85: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape84, R.shape([1, 10, 256])) lv22 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape85), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape86: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv22, R.shape([1, 1, 8, 256])) reshape87: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape86, R.shape([1, 1, 2048])) lv232 = R.call_tir(cls.NT_matmul11, (reshape87, model_layers_3_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv228_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv232, lv227_1, model_layers_3_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv229_1: R.Tensor((1, 1, 2048), dtype="float16") = lv228_1[1] rms_norm44: R.Tensor((1, 1, 2048), dtype="float16") = lv228_1[0] lv233 = R.call_tir(cls.NT_matmul12, (rms_norm44, model_layers_3_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split21: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv233, indices_or_sections=2, axis=-1) split_021: R.Tensor((1, 1, 16384), dtype="float16") = split21[0] split_121: R.Tensor((1, 1, 16384), dtype="float16") = split21[1] gelu21: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_021) mul23: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu21, split_121) lv234 = R.call_tir(cls.NT_matmul13, (mul23, model_layers_3_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv230_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv234, lv229_1, model_layers_4_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv231_1: R.Tensor((1, 1, 2048), dtype="float16") = lv230_1[1] rms_norm45: R.Tensor((1, 1, 2048), dtype="float16") = lv230_1[0] lv235 = R.call_tir(cls.NT_matmul10, (rms_norm45, model_layers_4_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape88: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv235, R.shape([1, 1, 10, 256])) reshape89: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape88, R.shape([1, 10, 256])) lv23 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape89), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape90: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv23, R.shape([1, 1, 8, 256])) reshape91: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape90, R.shape([1, 1, 2048])) lv236 = R.call_tir(cls.NT_matmul11, (reshape91, model_layers_4_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv232_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv236, lv231_1, model_layers_4_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv233_1: R.Tensor((1, 1, 2048), dtype="float16") = lv232_1[1] rms_norm46: R.Tensor((1, 1, 2048), dtype="float16") = lv232_1[0] lv237 = R.call_tir(cls.NT_matmul12, (rms_norm46, model_layers_4_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split22: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv237, indices_or_sections=2, axis=-1) split_022: R.Tensor((1, 1, 16384), dtype="float16") = split22[0] split_122: R.Tensor((1, 1, 16384), dtype="float16") = split22[1] gelu22: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_022) mul24: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu22, split_122) lv238 = R.call_tir(cls.NT_matmul13, (mul24, model_layers_4_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv234_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv238, lv233_1, model_layers_5_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv235_1: R.Tensor((1, 1, 2048), dtype="float16") = lv234_1[1] rms_norm47: R.Tensor((1, 1, 2048), dtype="float16") = lv234_1[0] lv239 = R.call_tir(cls.NT_matmul10, (rms_norm47, model_layers_5_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape92: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv239, R.shape([1, 1, 10, 256])) reshape93: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape92, R.shape([1, 10, 256])) lv24 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape93), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape94: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv24, R.shape([1, 1, 8, 256])) reshape95: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape94, R.shape([1, 1, 2048])) lv240 = R.call_tir(cls.NT_matmul11, (reshape95, model_layers_5_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv236_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv240, lv235_1, model_layers_5_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv237_1: R.Tensor((1, 1, 2048), dtype="float16") = lv236_1[1] rms_norm48: R.Tensor((1, 1, 2048), dtype="float16") = lv236_1[0] lv241 = R.call_tir(cls.NT_matmul12, (rms_norm48, model_layers_5_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split23: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv241, indices_or_sections=2, axis=-1) split_023: R.Tensor((1, 1, 16384), dtype="float16") = split23[0] split_123: R.Tensor((1, 1, 16384), dtype="float16") = split23[1] gelu23: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_023) mul25: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu23, split_123) lv242 = R.call_tir(cls.NT_matmul13, (mul25, model_layers_5_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv238_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv242, lv237_1, model_layers_6_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv239_1: R.Tensor((1, 1, 2048), dtype="float16") = lv238_1[1] rms_norm49: R.Tensor((1, 1, 2048), dtype="float16") = lv238_1[0] lv243 = R.call_tir(cls.NT_matmul10, (rms_norm49, model_layers_6_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape96: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv243, R.shape([1, 1, 10, 256])) reshape97: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape96, R.shape([1, 10, 256])) lv25 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape97), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape98: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv25, R.shape([1, 1, 8, 256])) reshape99: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape98, R.shape([1, 1, 2048])) lv244 = R.call_tir(cls.NT_matmul11, (reshape99, model_layers_6_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv240_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv244, lv239_1, model_layers_6_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv241_1: R.Tensor((1, 1, 2048), dtype="float16") = lv240_1[1] rms_norm50: R.Tensor((1, 1, 2048), dtype="float16") = lv240_1[0] lv245 = R.call_tir(cls.NT_matmul12, (rms_norm50, model_layers_6_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split24: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv245, indices_or_sections=2, axis=-1) split_024: R.Tensor((1, 1, 16384), dtype="float16") = split24[0] split_124: R.Tensor((1, 1, 16384), dtype="float16") = split24[1] gelu24: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_024) mul26: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu24, split_124) lv246 = R.call_tir(cls.NT_matmul13, (mul26, model_layers_6_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv242_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv246, lv241_1, model_layers_7_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv243_1: R.Tensor((1, 1, 2048), dtype="float16") = lv242_1[1] rms_norm51: R.Tensor((1, 1, 2048), dtype="float16") = lv242_1[0] lv247 = R.call_tir(cls.NT_matmul10, (rms_norm51, model_layers_7_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape100: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv247, R.shape([1, 1, 10, 256])) reshape101: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape100, R.shape([1, 10, 256])) lv26 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape101), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape102: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv26, R.shape([1, 1, 8, 256])) reshape103: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape102, R.shape([1, 1, 2048])) lv248 = R.call_tir(cls.NT_matmul11, (reshape103, model_layers_7_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv244_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv248, lv243_1, model_layers_7_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv245_1: R.Tensor((1, 1, 2048), dtype="float16") = lv244_1[1] rms_norm52: R.Tensor((1, 1, 2048), dtype="float16") = lv244_1[0] lv249 = R.call_tir(cls.NT_matmul12, (rms_norm52, model_layers_7_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split25: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv249, indices_or_sections=2, axis=-1) split_025: R.Tensor((1, 1, 16384), dtype="float16") = split25[0] split_125: R.Tensor((1, 1, 16384), dtype="float16") = split25[1] gelu25: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_025) mul27: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu25, split_125) lv250 = R.call_tir(cls.NT_matmul13, (mul27, model_layers_7_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv246_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv250, lv245_1, model_layers_8_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv247_1: R.Tensor((1, 1, 2048), dtype="float16") = lv246_1[1] rms_norm53: R.Tensor((1, 1, 2048), dtype="float16") = lv246_1[0] lv251 = R.call_tir(cls.NT_matmul10, (rms_norm53, model_layers_8_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape104: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv251, R.shape([1, 1, 10, 256])) reshape105: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape104, R.shape([1, 10, 256])) lv27 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape105), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape106: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv27, R.shape([1, 1, 8, 256])) reshape107: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape106, R.shape([1, 1, 2048])) lv252 = R.call_tir(cls.NT_matmul11, (reshape107, model_layers_8_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv248_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv252, lv247_1, model_layers_8_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv249_1: R.Tensor((1, 1, 2048), dtype="float16") = lv248_1[1] rms_norm54: R.Tensor((1, 1, 2048), dtype="float16") = lv248_1[0] lv253 = R.call_tir(cls.NT_matmul12, (rms_norm54, model_layers_8_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split26: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv253, indices_or_sections=2, axis=-1) split_026: R.Tensor((1, 1, 16384), dtype="float16") = split26[0] split_126: R.Tensor((1, 1, 16384), dtype="float16") = split26[1] gelu26: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_026) mul28: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu26, split_126) lv254 = R.call_tir(cls.NT_matmul13, (mul28, model_layers_8_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv250_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv254, lv249_1, model_layers_9_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv251_1: R.Tensor((1, 1, 2048), dtype="float16") = lv250_1[1] rms_norm55: R.Tensor((1, 1, 2048), dtype="float16") = lv250_1[0] lv255 = R.call_tir(cls.NT_matmul10, (rms_norm55, model_layers_9_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape108: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv255, R.shape([1, 1, 10, 256])) reshape109: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape108, R.shape([1, 10, 256])) lv28 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape109), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape110: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv28, R.shape([1, 1, 8, 256])) reshape111: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape110, R.shape([1, 1, 2048])) lv256 = R.call_tir(cls.NT_matmul11, (reshape111, model_layers_9_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv252_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv256, lv251_1, model_layers_9_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv253_1: R.Tensor((1, 1, 2048), dtype="float16") = lv252_1[1] rms_norm56: R.Tensor((1, 1, 2048), dtype="float16") = lv252_1[0] lv257 = R.call_tir(cls.NT_matmul12, (rms_norm56, model_layers_9_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split27: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv257, indices_or_sections=2, axis=-1) split_027: R.Tensor((1, 1, 16384), dtype="float16") = split27[0] split_127: R.Tensor((1, 1, 16384), dtype="float16") = split27[1] gelu27: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_027) mul29: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu27, split_127) lv258 = R.call_tir(cls.NT_matmul13, (mul29, model_layers_9_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv254_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv258, lv253_1, model_layers_10_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv255_1: R.Tensor((1, 1, 2048), dtype="float16") = lv254_1[1] rms_norm57: R.Tensor((1, 1, 2048), dtype="float16") = lv254_1[0] lv259 = R.call_tir(cls.NT_matmul10, (rms_norm57, model_layers_10_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape112: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv259, R.shape([1, 1, 10, 256])) reshape113: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape112, R.shape([1, 10, 256])) lv29 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape113), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape114: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv29, R.shape([1, 1, 8, 256])) reshape115: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape114, R.shape([1, 1, 2048])) lv260 = R.call_tir(cls.NT_matmul11, (reshape115, model_layers_10_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv256_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv260, lv255_1, model_layers_10_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv257_1: R.Tensor((1, 1, 2048), dtype="float16") = lv256_1[1] rms_norm58: R.Tensor((1, 1, 2048), dtype="float16") = lv256_1[0] lv261 = R.call_tir(cls.NT_matmul12, (rms_norm58, model_layers_10_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split28: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv261, indices_or_sections=2, axis=-1) split_028: R.Tensor((1, 1, 16384), dtype="float16") = split28[0] split_128: R.Tensor((1, 1, 16384), dtype="float16") = split28[1] gelu28: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_028) mul30: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu28, split_128) lv262 = R.call_tir(cls.NT_matmul13, (mul30, model_layers_10_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv258_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv262, lv257_1, model_layers_11_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv259_1: R.Tensor((1, 1, 2048), dtype="float16") = lv258_1[1] rms_norm59: R.Tensor((1, 1, 2048), dtype="float16") = lv258_1[0] lv263 = R.call_tir(cls.NT_matmul10, (rms_norm59, model_layers_11_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape116: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv263, R.shape([1, 1, 10, 256])) reshape117: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape116, R.shape([1, 10, 256])) lv30 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape117), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape118: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv30, R.shape([1, 1, 8, 256])) reshape119: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape118, R.shape([1, 1, 2048])) lv264 = R.call_tir(cls.NT_matmul11, (reshape119, model_layers_11_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv260_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv264, lv259_1, model_layers_11_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv261_1: R.Tensor((1, 1, 2048), dtype="float16") = lv260_1[1] rms_norm60: R.Tensor((1, 1, 2048), dtype="float16") = lv260_1[0] lv265 = R.call_tir(cls.NT_matmul12, (rms_norm60, model_layers_11_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split29: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv265, indices_or_sections=2, axis=-1) split_029: R.Tensor((1, 1, 16384), dtype="float16") = split29[0] split_129: R.Tensor((1, 1, 16384), dtype="float16") = split29[1] gelu29: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_029) mul31: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu29, split_129) lv266 = R.call_tir(cls.NT_matmul13, (mul31, model_layers_11_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv262_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv266, lv261_1, model_layers_12_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv263_1: R.Tensor((1, 1, 2048), dtype="float16") = lv262_1[1] rms_norm61: R.Tensor((1, 1, 2048), dtype="float16") = lv262_1[0] lv267 = R.call_tir(cls.NT_matmul10, (rms_norm61, model_layers_12_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape120: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv267, R.shape([1, 1, 10, 256])) reshape121: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape120, R.shape([1, 10, 256])) lv31 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape121), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape122: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv31, R.shape([1, 1, 8, 256])) reshape123: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape122, R.shape([1, 1, 2048])) lv268 = R.call_tir(cls.NT_matmul11, (reshape123, model_layers_12_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv264_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv268, lv263_1, model_layers_12_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv265_1: R.Tensor((1, 1, 2048), dtype="float16") = lv264_1[1] rms_norm62: R.Tensor((1, 1, 2048), dtype="float16") = lv264_1[0] lv269 = R.call_tir(cls.NT_matmul12, (rms_norm62, model_layers_12_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split30: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv269, indices_or_sections=2, axis=-1) split_030: R.Tensor((1, 1, 16384), dtype="float16") = split30[0] split_130: R.Tensor((1, 1, 16384), dtype="float16") = split30[1] gelu30: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_030) mul32: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu30, split_130) lv270 = R.call_tir(cls.NT_matmul13, (mul32, model_layers_12_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv266_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv270, lv265_1, model_layers_13_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv267_1: R.Tensor((1, 1, 2048), dtype="float16") = lv266_1[1] rms_norm63: R.Tensor((1, 1, 2048), dtype="float16") = lv266_1[0] lv271 = R.call_tir(cls.NT_matmul10, (rms_norm63, model_layers_13_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape124: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv271, R.shape([1, 1, 10, 256])) reshape125: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape124, R.shape([1, 10, 256])) lv32 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape125), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape126: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv32, R.shape([1, 1, 8, 256])) reshape127: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape126, R.shape([1, 1, 2048])) lv272 = R.call_tir(cls.NT_matmul11, (reshape127, model_layers_13_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv268_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv272, lv267_1, model_layers_13_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv269_1: R.Tensor((1, 1, 2048), dtype="float16") = lv268_1[1] rms_norm64: R.Tensor((1, 1, 2048), dtype="float16") = lv268_1[0] lv273 = R.call_tir(cls.NT_matmul12, (rms_norm64, model_layers_13_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split31: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv273, indices_or_sections=2, axis=-1) split_031: R.Tensor((1, 1, 16384), dtype="float16") = split31[0] split_131: R.Tensor((1, 1, 16384), dtype="float16") = split31[1] gelu31: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_031) mul33: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu31, split_131) lv274 = R.call_tir(cls.NT_matmul13, (mul33, model_layers_13_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv270_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv274, lv269_1, model_layers_14_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv271_1: R.Tensor((1, 1, 2048), dtype="float16") = lv270_1[1] rms_norm65: R.Tensor((1, 1, 2048), dtype="float16") = lv270_1[0] lv275 = R.call_tir(cls.NT_matmul10, (rms_norm65, model_layers_14_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape128: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv275, R.shape([1, 1, 10, 256])) reshape129: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape128, R.shape([1, 10, 256])) lv33 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape129), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape130: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv33, R.shape([1, 1, 8, 256])) reshape131: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape130, R.shape([1, 1, 2048])) lv276 = R.call_tir(cls.NT_matmul11, (reshape131, model_layers_14_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv272_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv276, lv271_1, model_layers_14_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv273_1: R.Tensor((1, 1, 2048), dtype="float16") = lv272_1[1] rms_norm66: R.Tensor((1, 1, 2048), dtype="float16") = lv272_1[0] lv277 = R.call_tir(cls.NT_matmul12, (rms_norm66, model_layers_14_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split32: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv277, indices_or_sections=2, axis=-1) split_032: R.Tensor((1, 1, 16384), dtype="float16") = split32[0] split_132: R.Tensor((1, 1, 16384), dtype="float16") = split32[1] gelu32: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_032) mul34: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu32, split_132) lv278 = R.call_tir(cls.NT_matmul13, (mul34, model_layers_14_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv274_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv278, lv273_1, model_layers_15_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv275_1: R.Tensor((1, 1, 2048), dtype="float16") = lv274_1[1] rms_norm67: R.Tensor((1, 1, 2048), dtype="float16") = lv274_1[0] lv279 = R.call_tir(cls.NT_matmul10, (rms_norm67, model_layers_15_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape132: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv279, R.shape([1, 1, 10, 256])) reshape133: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape132, R.shape([1, 10, 256])) lv34 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape133), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape134: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv34, R.shape([1, 1, 8, 256])) reshape135: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape134, R.shape([1, 1, 2048])) lv280 = R.call_tir(cls.NT_matmul11, (reshape135, model_layers_15_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv276_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv280, lv275_1, model_layers_15_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv277_1: R.Tensor((1, 1, 2048), dtype="float16") = lv276_1[1] rms_norm68: R.Tensor((1, 1, 2048), dtype="float16") = lv276_1[0] lv281 = R.call_tir(cls.NT_matmul12, (rms_norm68, model_layers_15_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split33: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv281, indices_or_sections=2, axis=-1) split_033: R.Tensor((1, 1, 16384), dtype="float16") = split33[0] split_133: R.Tensor((1, 1, 16384), dtype="float16") = split33[1] gelu33: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_033) mul35: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu33, split_133) lv282 = R.call_tir(cls.NT_matmul13, (mul35, model_layers_15_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv278_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv282, lv277_1, model_layers_16_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv279_1: R.Tensor((1, 1, 2048), dtype="float16") = lv278_1[1] rms_norm69: R.Tensor((1, 1, 2048), dtype="float16") = lv278_1[0] lv283 = R.call_tir(cls.NT_matmul10, (rms_norm69, model_layers_16_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape136: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv283, R.shape([1, 1, 10, 256])) reshape137: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape136, R.shape([1, 10, 256])) lv35 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape137), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape138: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv35, R.shape([1, 1, 8, 256])) reshape139: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape138, R.shape([1, 1, 2048])) lv284 = R.call_tir(cls.NT_matmul11, (reshape139, model_layers_16_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv280_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv284, lv279_1, model_layers_16_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv281_1: R.Tensor((1, 1, 2048), dtype="float16") = lv280_1[1] rms_norm70: R.Tensor((1, 1, 2048), dtype="float16") = lv280_1[0] lv285 = R.call_tir(cls.NT_matmul12, (rms_norm70, model_layers_16_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split34: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv285, indices_or_sections=2, axis=-1) split_034: R.Tensor((1, 1, 16384), dtype="float16") = split34[0] split_134: R.Tensor((1, 1, 16384), dtype="float16") = split34[1] gelu34: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_034) mul36: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu34, split_134) lv286 = R.call_tir(cls.NT_matmul13, (mul36, model_layers_16_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv282_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv286, lv281_1, model_layers_17_input_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv283_1: R.Tensor((1, 1, 2048), dtype="float16") = lv282_1[1] rms_norm71: R.Tensor((1, 1, 2048), dtype="float16") = lv282_1[0] lv287 = R.call_tir(cls.NT_matmul10, (rms_norm71, model_layers_17_self_attn_qkv_proj_weight2), out_sinfo=R.Tensor((1, 1, 2560), dtype="float16")) reshape140: R.Tensor((1, 1, 10, 256), dtype="float16") = R.reshape(lv287, R.shape([1, 1, 10, 256])) reshape141: R.Tensor((1, 10, 256), dtype="float16") = R.reshape(reshape140, R.shape([1, 10, 256])) lv36 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape141), out_sinfo=R.Tensor((1, 8, 256), dtype="float16")) reshape142: R.Tensor((1, 1, 8, 256), dtype="float16") = R.reshape(lv36, R.shape([1, 1, 8, 256])) reshape143: R.Tensor((1, 1, 2048), dtype="float16") = R.reshape(reshape142, R.shape([1, 1, 2048])) lv288 = R.call_tir(cls.NT_matmul11, (reshape143, model_layers_17_self_attn_o_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv284_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv288, lv283_1, model_layers_17_post_attention_layernorm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) lv285_1: R.Tensor((1, 1, 2048), dtype="float16") = lv284_1[1] rms_norm72: R.Tensor((1, 1, 2048), dtype="float16") = lv284_1[0] lv289 = R.call_tir(cls.NT_matmul12, (rms_norm72, model_layers_17_mlp_gate_up_proj_weight2), out_sinfo=R.Tensor((1, 1, 32768), dtype="float16")) split35: R.Tuple(R.Tensor((1, 1, 16384), dtype="float16"), R.Tensor((1, 1, 16384), dtype="float16")) = R.split(lv289, indices_or_sections=2, axis=-1) split_035: R.Tensor((1, 1, 16384), dtype="float16") = split35[0] split_135: R.Tensor((1, 1, 16384), dtype="float16") = split35[1] gelu35: R.Tensor((1, 1, 16384), dtype="float16") = R.nn.gelu_tanh(split_035) mul37: R.Tensor((1, 1, 16384), dtype="float16") = R.multiply(gelu35, split_135) lv290 = R.call_tir(cls.NT_matmul13, (mul37, model_layers_17_mlp_down_proj_weight2), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv286_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv290, lv285_1, model_norm_weight2), out_sinfo=[R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")]) rms_norm73: R.Tensor((1, 1, 2048), dtype="float16") = lv286_1[0] lv291 = R.call_tir(cls.NT_matmul14, (rms_norm73, model_embed_tokens_weight2), out_sinfo=R.Tensor((1, 1, vocab_size), dtype="float32")) gv2: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = lv291, paged_kv_cache R.output(gv2) return gv2 @R.function def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tensor(("seq_len", 2048), dtype="float16"): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) with R.dataflow(): model_embed_tokens_weight: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] take: R.Tensor((seq_len, 2048), dtype="float16") = R.take(model_embed_tokens_weight, input_ids, axis=0) gv: R.Tensor((seq_len, 2048), dtype="float16") = take R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul(model_layers_0_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16"), rms_norm111: R.Tensor(("batch_size", 1, 2048), dtype="float16")) -> R.Tensor(("batch_size", 1, 2560), dtype="float16"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims219: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight4, axes=None) gv: R.Tensor((batch_size, 1, 2560), dtype="float16") = R.matmul(rms_norm111, permute_dims219, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul1(model_layers_0_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16"), reshape219: R.Tensor(("batch_size", 1, 2048), dtype="float16")) -> R.Tensor(("batch_size", 1, 2048), dtype="float16"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims220: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight4, axes=None) gv: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(reshape219, permute_dims220, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul10(model_layers_0_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16"), rms_norm37: R.Tensor((1, 1, 2048), dtype="float16")) -> R.Tensor((1, 1, 2560), dtype="float16"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims73: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight2, axes=None) gv: R.Tensor((1, 1, 2560), dtype="float16") = R.matmul(rms_norm37, permute_dims73, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul11(model_layers_0_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16"), reshape75: R.Tensor((1, 1, 2048), dtype="float16")) -> R.Tensor((1, 1, 2048), dtype="float16"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims74: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight2, axes=None) gv: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(reshape75, permute_dims74, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul12(model_layers_0_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16"), rms_norm38: R.Tensor((1, 1, 2048), dtype="float16")) -> R.Tensor((1, 1, 32768), dtype="float16"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims75: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight2, axes=None) gv: R.Tensor((1, 1, 32768), dtype="float16") = R.matmul(rms_norm38, permute_dims75, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul13(model_layers_0_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16"), mul20: R.Tensor((1, 1, 16384), dtype="float16")) -> R.Tensor((1, 1, 2048), dtype="float16"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims76: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight2, axes=None) gv: R.Tensor((1, 1, 2048), dtype="float16") = R.matmul(mul20, permute_dims76, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul14(model_embed_tokens_weight2: R.Tensor(("vocab_size", 2048), dtype="float16"), rms_norm73: R.Tensor((1, 1, 2048), dtype="float16")) -> R.Tensor((1, 1, "vocab_size"), dtype="float32"): vocab_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims145: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight2, axes=None) gv: R.Tensor((1, 1, vocab_size), dtype="float32") = R.matmul(rms_norm73, permute_dims145, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul2(model_layers_0_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16"), rms_norm112: R.Tensor(("batch_size", 1, 2048), dtype="float16")) -> R.Tensor(("batch_size", 1, 32768), dtype="float16"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims221: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight4, axes=None) gv: R.Tensor((batch_size, 1, 32768), dtype="float16") = R.matmul(rms_norm112, permute_dims221, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul3(model_layers_0_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16"), mul58: R.Tensor(("batch_size", 1, 16384), dtype="float16")) -> R.Tensor(("batch_size", 1, 2048), dtype="float16"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims222: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight4, axes=None) gv: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.matmul(mul58, permute_dims222, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul4(model_embed_tokens_weight4: R.Tensor(("vocab_size", 2048), dtype="float16"), rms_norm147: R.Tensor(("batch_size", 1, 2048), dtype="float16")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims291: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight4, axes=None) gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.matmul(rms_norm147, permute_dims291, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul5(model_layers_0_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16"), rms_norm74: R.Tensor((1, "seq_len", 2048), dtype="float16")) -> R.Tensor((1, "seq_len", 2560), dtype="float16"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims146: R.Tensor((2048, 2560), dtype="float16") = R.permute_dims(model_layers_0_self_attn_qkv_proj_weight3, axes=None) gv: R.Tensor((1, seq_len, 2560), dtype="float16") = R.matmul(rms_norm74, permute_dims146, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul6(model_layers_0_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16"), reshape147: R.Tensor((1, "seq_len", 2048), dtype="float16")) -> R.Tensor((1, "seq_len", 2048), dtype="float16"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims147: R.Tensor((2048, 2048), dtype="float16") = R.permute_dims(model_layers_0_self_attn_o_proj_weight3, axes=None) gv: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(reshape147, permute_dims147, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul7(model_layers_0_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16"), rms_norm75: R.Tensor((1, "seq_len", 2048), dtype="float16")) -> R.Tensor((1, "seq_len", 32768), dtype="float16"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims148: R.Tensor((2048, 32768), dtype="float16") = R.permute_dims(model_layers_0_mlp_gate_up_proj_weight3, axes=None) gv: R.Tensor((1, seq_len, 32768), dtype="float16") = R.matmul(rms_norm75, permute_dims148, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul8(model_layers_0_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16"), mul39: R.Tensor((1, "seq_len", 16384), dtype="float16")) -> R.Tensor((1, "seq_len", 2048), dtype="float16"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims149: R.Tensor((16384, 2048), dtype="float16") = R.permute_dims(model_layers_0_mlp_down_proj_weight3, axes=None) gv: R.Tensor((1, seq_len, 2048), dtype="float16") = R.matmul(mul39, permute_dims149, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul9(model_embed_tokens_weight3: R.Tensor(("vocab_size", 2048), dtype="float16"), take1: R.Tensor((1, "batch_size", 2048), dtype="float16")) -> R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims218: R.Tensor((2048, vocab_size), dtype="float16") = R.permute_dims(model_embed_tokens_weight3, axes=None) gv: R.Tensor((1, batch_size, vocab_size), dtype="float32") = R.matmul(take1, permute_dims218, out_dtype="float32") R.output(gv) return gv @R.function def prefill(input_embed: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): model_embed_tokens_weight1: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] model_layers_0_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] model_layers_0_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] model_layers_0_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] model_layers_0_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] model_layers_0_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[5] model_layers_0_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[6] model_layers_1_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] model_layers_1_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] model_layers_1_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] model_layers_1_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] model_layers_1_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[11] model_layers_1_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[12] model_layers_2_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] model_layers_2_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] model_layers_2_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] model_layers_2_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] model_layers_2_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[17] model_layers_2_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[18] model_layers_3_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] model_layers_3_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] model_layers_3_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] model_layers_3_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] model_layers_3_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[23] model_layers_3_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[24] model_layers_4_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] model_layers_4_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] model_layers_4_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] model_layers_4_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] model_layers_4_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[29] model_layers_4_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[30] model_layers_5_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] model_layers_5_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] model_layers_5_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] model_layers_5_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] model_layers_5_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[35] model_layers_5_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[36] model_layers_6_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] model_layers_6_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] model_layers_6_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] model_layers_6_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] model_layers_6_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[41] model_layers_6_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[42] model_layers_7_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] model_layers_7_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] model_layers_7_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] model_layers_7_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] model_layers_7_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[47] model_layers_7_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[48] model_layers_8_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] model_layers_8_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] model_layers_8_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] model_layers_8_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] model_layers_8_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[53] model_layers_8_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[54] model_layers_9_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] model_layers_9_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] model_layers_9_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] model_layers_9_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] model_layers_9_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[59] model_layers_9_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[60] model_layers_10_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] model_layers_10_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] model_layers_10_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] model_layers_10_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] model_layers_10_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[65] model_layers_10_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[66] model_layers_11_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] model_layers_11_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] model_layers_11_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] model_layers_11_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] model_layers_11_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[71] model_layers_11_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[72] model_layers_12_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] model_layers_12_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] model_layers_12_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] model_layers_12_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] model_layers_12_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[77] model_layers_12_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[78] model_layers_13_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] model_layers_13_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] model_layers_13_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] model_layers_13_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] model_layers_13_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[83] model_layers_13_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[84] model_layers_14_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] model_layers_14_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] model_layers_14_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] model_layers_14_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] model_layers_14_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[89] model_layers_14_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[90] model_layers_15_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] model_layers_15_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] model_layers_15_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] model_layers_15_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] model_layers_15_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[95] model_layers_15_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[96] model_layers_16_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] model_layers_16_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] model_layers_16_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] model_layers_16_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] model_layers_16_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[101] model_layers_16_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[102] model_layers_17_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] model_layers_17_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] model_layers_17_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] model_layers_17_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] model_layers_17_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[107] model_layers_17_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[108] model_norm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[109] mul: R.Tensor((1, seq_len, 2048), dtype="float16") = R.multiply(input_embed, R.const(45.25, "float16")) rms_norm: R.Tensor((1, seq_len, 2048), dtype="float16") = R.nn.rms_norm(mul, model_layers_0_input_layernorm_weight1, axes=[-1], epsilon=9.9999999999999995e-07) lv292 = R.call_tir(cls.NT_matmul5, (rms_norm, model_layers_0_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv292, R.shape([1, seq_len, 10, 256])) reshape1: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape, R.shape([seq_len, 10, 256])) lv = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape2: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv, R.shape([1, seq_len, 8, 256])) reshape3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape2, R.shape([1, seq_len, 2048])) lv293 = R.call_tir(cls.NT_matmul6, (reshape3, model_layers_0_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv288 = R.call_tir(cls.fuse_add_norm_prefill, (lv293, mul, model_layers_0_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv289: R.Tensor((1, seq_len, 2048), dtype="float16") = lv288[1] rms_norm1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv288[0] lv294 = R.call_tir(cls.NT_matmul7, (rms_norm1, model_layers_0_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv294, indices_or_sections=2, axis=-1) split_0: R.Tensor((1, seq_len, 16384), dtype="float16") = split[0] split_1: R.Tensor((1, seq_len, 16384), dtype="float16") = split[1] gelu: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_0) mul1: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu, split_1) lv295 = R.call_tir(cls.NT_matmul8, (mul1, model_layers_0_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv290 = R.call_tir(cls.fuse_add_norm_prefill, (lv295, lv289, model_layers_1_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv291: R.Tensor((1, seq_len, 2048), dtype="float16") = lv290[1] rms_norm2: R.Tensor((1, seq_len, 2048), dtype="float16") = lv290[0] lv296 = R.call_tir(cls.NT_matmul5, (rms_norm2, model_layers_1_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape4: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv296, R.shape([1, seq_len, 10, 256])) reshape5: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape4, R.shape([seq_len, 10, 256])) lv1 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape6: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv1, R.shape([1, seq_len, 8, 256])) reshape7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape6, R.shape([1, seq_len, 2048])) lv297 = R.call_tir(cls.NT_matmul6, (reshape7, model_layers_1_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv292_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv297, lv291, model_layers_1_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv293_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv292_1[1] rms_norm3: R.Tensor((1, seq_len, 2048), dtype="float16") = lv292_1[0] lv298 = R.call_tir(cls.NT_matmul7, (rms_norm3, model_layers_1_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split1: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv298, indices_or_sections=2, axis=-1) split_01: R.Tensor((1, seq_len, 16384), dtype="float16") = split1[0] split_11: R.Tensor((1, seq_len, 16384), dtype="float16") = split1[1] gelu1: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_01) mul2: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu1, split_11) lv299 = R.call_tir(cls.NT_matmul8, (mul2, model_layers_1_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv294_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv299, lv293_1, model_layers_2_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv295_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv294_1[1] rms_norm4: R.Tensor((1, seq_len, 2048), dtype="float16") = lv294_1[0] lv300 = R.call_tir(cls.NT_matmul5, (rms_norm4, model_layers_2_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape8: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv300, R.shape([1, seq_len, 10, 256])) reshape9: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape8, R.shape([seq_len, 10, 256])) lv2 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape10: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv2, R.shape([1, seq_len, 8, 256])) reshape11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape10, R.shape([1, seq_len, 2048])) lv301 = R.call_tir(cls.NT_matmul6, (reshape11, model_layers_2_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv296_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv301, lv295_1, model_layers_2_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv297_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv296_1[1] rms_norm5: R.Tensor((1, seq_len, 2048), dtype="float16") = lv296_1[0] lv302 = R.call_tir(cls.NT_matmul7, (rms_norm5, model_layers_2_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split2: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv302, indices_or_sections=2, axis=-1) split_02: R.Tensor((1, seq_len, 16384), dtype="float16") = split2[0] split_12: R.Tensor((1, seq_len, 16384), dtype="float16") = split2[1] gelu2: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_02) mul3: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu2, split_12) lv303 = R.call_tir(cls.NT_matmul8, (mul3, model_layers_2_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv298_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv303, lv297_1, model_layers_3_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv299_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv298_1[1] rms_norm6: R.Tensor((1, seq_len, 2048), dtype="float16") = lv298_1[0] lv304 = R.call_tir(cls.NT_matmul5, (rms_norm6, model_layers_3_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape12: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv304, R.shape([1, seq_len, 10, 256])) reshape13: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape12, R.shape([seq_len, 10, 256])) lv3 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape14: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv3, R.shape([1, seq_len, 8, 256])) reshape15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape14, R.shape([1, seq_len, 2048])) lv305 = R.call_tir(cls.NT_matmul6, (reshape15, model_layers_3_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv300_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv305, lv299_1, model_layers_3_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv301_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv300_1[1] rms_norm7: R.Tensor((1, seq_len, 2048), dtype="float16") = lv300_1[0] lv306 = R.call_tir(cls.NT_matmul7, (rms_norm7, model_layers_3_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split3: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv306, indices_or_sections=2, axis=-1) split_03: R.Tensor((1, seq_len, 16384), dtype="float16") = split3[0] split_13: R.Tensor((1, seq_len, 16384), dtype="float16") = split3[1] gelu3: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_03) mul4: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu3, split_13) lv307 = R.call_tir(cls.NT_matmul8, (mul4, model_layers_3_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv302_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv307, lv301_1, model_layers_4_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv303_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv302_1[1] rms_norm8: R.Tensor((1, seq_len, 2048), dtype="float16") = lv302_1[0] lv308 = R.call_tir(cls.NT_matmul5, (rms_norm8, model_layers_4_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape16: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv308, R.shape([1, seq_len, 10, 256])) reshape17: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape16, R.shape([seq_len, 10, 256])) lv4 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape18: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv4, R.shape([1, seq_len, 8, 256])) reshape19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape18, R.shape([1, seq_len, 2048])) lv309 = R.call_tir(cls.NT_matmul6, (reshape19, model_layers_4_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv304_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv309, lv303_1, model_layers_4_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv305_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv304_1[1] rms_norm9: R.Tensor((1, seq_len, 2048), dtype="float16") = lv304_1[0] lv310 = R.call_tir(cls.NT_matmul7, (rms_norm9, model_layers_4_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split4: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv310, indices_or_sections=2, axis=-1) split_04: R.Tensor((1, seq_len, 16384), dtype="float16") = split4[0] split_14: R.Tensor((1, seq_len, 16384), dtype="float16") = split4[1] gelu4: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_04) mul5: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu4, split_14) lv311 = R.call_tir(cls.NT_matmul8, (mul5, model_layers_4_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv306_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv311, lv305_1, model_layers_5_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv307_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv306_1[1] rms_norm10: R.Tensor((1, seq_len, 2048), dtype="float16") = lv306_1[0] lv312 = R.call_tir(cls.NT_matmul5, (rms_norm10, model_layers_5_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape20: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv312, R.shape([1, seq_len, 10, 256])) reshape21: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape20, R.shape([seq_len, 10, 256])) lv5 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape22: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv5, R.shape([1, seq_len, 8, 256])) reshape23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape22, R.shape([1, seq_len, 2048])) lv313 = R.call_tir(cls.NT_matmul6, (reshape23, model_layers_5_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv308_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv313, lv307_1, model_layers_5_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv309_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv308_1[1] rms_norm11: R.Tensor((1, seq_len, 2048), dtype="float16") = lv308_1[0] lv314 = R.call_tir(cls.NT_matmul7, (rms_norm11, model_layers_5_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split5: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv314, indices_or_sections=2, axis=-1) split_05: R.Tensor((1, seq_len, 16384), dtype="float16") = split5[0] split_15: R.Tensor((1, seq_len, 16384), dtype="float16") = split5[1] gelu5: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_05) mul6: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu5, split_15) lv315 = R.call_tir(cls.NT_matmul8, (mul6, model_layers_5_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv310_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv315, lv309_1, model_layers_6_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv311_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv310_1[1] rms_norm12: R.Tensor((1, seq_len, 2048), dtype="float16") = lv310_1[0] lv316 = R.call_tir(cls.NT_matmul5, (rms_norm12, model_layers_6_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape24: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv316, R.shape([1, seq_len, 10, 256])) reshape25: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape24, R.shape([seq_len, 10, 256])) lv6 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape25), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape26: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv6, R.shape([1, seq_len, 8, 256])) reshape27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape26, R.shape([1, seq_len, 2048])) lv317 = R.call_tir(cls.NT_matmul6, (reshape27, model_layers_6_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv312_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv317, lv311_1, model_layers_6_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv313_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv312_1[1] rms_norm13: R.Tensor((1, seq_len, 2048), dtype="float16") = lv312_1[0] lv318 = R.call_tir(cls.NT_matmul7, (rms_norm13, model_layers_6_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split6: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv318, indices_or_sections=2, axis=-1) split_06: R.Tensor((1, seq_len, 16384), dtype="float16") = split6[0] split_16: R.Tensor((1, seq_len, 16384), dtype="float16") = split6[1] gelu6: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_06) mul7: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu6, split_16) lv319 = R.call_tir(cls.NT_matmul8, (mul7, model_layers_6_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv314_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv319, lv313_1, model_layers_7_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv315_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv314_1[1] rms_norm14: R.Tensor((1, seq_len, 2048), dtype="float16") = lv314_1[0] lv320 = R.call_tir(cls.NT_matmul5, (rms_norm14, model_layers_7_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape28: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv320, R.shape([1, seq_len, 10, 256])) reshape29: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape28, R.shape([seq_len, 10, 256])) lv7 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape29), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape30: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv7, R.shape([1, seq_len, 8, 256])) reshape31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape30, R.shape([1, seq_len, 2048])) lv321 = R.call_tir(cls.NT_matmul6, (reshape31, model_layers_7_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv316_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv321, lv315_1, model_layers_7_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv317_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv316_1[1] rms_norm15: R.Tensor((1, seq_len, 2048), dtype="float16") = lv316_1[0] lv322 = R.call_tir(cls.NT_matmul7, (rms_norm15, model_layers_7_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split7: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv322, indices_or_sections=2, axis=-1) split_07: R.Tensor((1, seq_len, 16384), dtype="float16") = split7[0] split_17: R.Tensor((1, seq_len, 16384), dtype="float16") = split7[1] gelu7: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_07) mul8: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu7, split_17) lv323 = R.call_tir(cls.NT_matmul8, (mul8, model_layers_7_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv318_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv323, lv317_1, model_layers_8_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv319_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv318_1[1] rms_norm16: R.Tensor((1, seq_len, 2048), dtype="float16") = lv318_1[0] lv324 = R.call_tir(cls.NT_matmul5, (rms_norm16, model_layers_8_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape32: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv324, R.shape([1, seq_len, 10, 256])) reshape33: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape32, R.shape([seq_len, 10, 256])) lv8 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape33), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape34: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv8, R.shape([1, seq_len, 8, 256])) reshape35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape34, R.shape([1, seq_len, 2048])) lv325 = R.call_tir(cls.NT_matmul6, (reshape35, model_layers_8_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv320_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv325, lv319_1, model_layers_8_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv321_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv320_1[1] rms_norm17: R.Tensor((1, seq_len, 2048), dtype="float16") = lv320_1[0] lv326 = R.call_tir(cls.NT_matmul7, (rms_norm17, model_layers_8_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split8: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv326, indices_or_sections=2, axis=-1) split_08: R.Tensor((1, seq_len, 16384), dtype="float16") = split8[0] split_18: R.Tensor((1, seq_len, 16384), dtype="float16") = split8[1] gelu8: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_08) mul9: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu8, split_18) lv327 = R.call_tir(cls.NT_matmul8, (mul9, model_layers_8_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv322_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv327, lv321_1, model_layers_9_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv323_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv322_1[1] rms_norm18: R.Tensor((1, seq_len, 2048), dtype="float16") = lv322_1[0] lv328 = R.call_tir(cls.NT_matmul5, (rms_norm18, model_layers_9_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape36: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv328, R.shape([1, seq_len, 10, 256])) reshape37: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape36, R.shape([seq_len, 10, 256])) lv9 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape37), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape38: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv9, R.shape([1, seq_len, 8, 256])) reshape39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape38, R.shape([1, seq_len, 2048])) lv329 = R.call_tir(cls.NT_matmul6, (reshape39, model_layers_9_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv324_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv329, lv323_1, model_layers_9_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv325_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv324_1[1] rms_norm19: R.Tensor((1, seq_len, 2048), dtype="float16") = lv324_1[0] lv330 = R.call_tir(cls.NT_matmul7, (rms_norm19, model_layers_9_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split9: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv330, indices_or_sections=2, axis=-1) split_09: R.Tensor((1, seq_len, 16384), dtype="float16") = split9[0] split_19: R.Tensor((1, seq_len, 16384), dtype="float16") = split9[1] gelu9: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_09) mul10: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu9, split_19) lv331 = R.call_tir(cls.NT_matmul8, (mul10, model_layers_9_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv326_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv331, lv325_1, model_layers_10_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv327_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv326_1[1] rms_norm20: R.Tensor((1, seq_len, 2048), dtype="float16") = lv326_1[0] lv332 = R.call_tir(cls.NT_matmul5, (rms_norm20, model_layers_10_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape40: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv332, R.shape([1, seq_len, 10, 256])) reshape41: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape40, R.shape([seq_len, 10, 256])) lv10 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape41), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape42: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv10, R.shape([1, seq_len, 8, 256])) reshape43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape42, R.shape([1, seq_len, 2048])) lv333 = R.call_tir(cls.NT_matmul6, (reshape43, model_layers_10_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv328_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv333, lv327_1, model_layers_10_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv329_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv328_1[1] rms_norm21: R.Tensor((1, seq_len, 2048), dtype="float16") = lv328_1[0] lv334 = R.call_tir(cls.NT_matmul7, (rms_norm21, model_layers_10_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split10: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv334, indices_or_sections=2, axis=-1) split_010: R.Tensor((1, seq_len, 16384), dtype="float16") = split10[0] split_110: R.Tensor((1, seq_len, 16384), dtype="float16") = split10[1] gelu10: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_010) mul11: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu10, split_110) lv335 = R.call_tir(cls.NT_matmul8, (mul11, model_layers_10_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv330_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv335, lv329_1, model_layers_11_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv331_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv330_1[1] rms_norm22: R.Tensor((1, seq_len, 2048), dtype="float16") = lv330_1[0] lv336 = R.call_tir(cls.NT_matmul5, (rms_norm22, model_layers_11_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape44: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv336, R.shape([1, seq_len, 10, 256])) reshape45: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape44, R.shape([seq_len, 10, 256])) lv11 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape45), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape46: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv11, R.shape([1, seq_len, 8, 256])) reshape47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape46, R.shape([1, seq_len, 2048])) lv337 = R.call_tir(cls.NT_matmul6, (reshape47, model_layers_11_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv332_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv337, lv331_1, model_layers_11_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv333_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv332_1[1] rms_norm23: R.Tensor((1, seq_len, 2048), dtype="float16") = lv332_1[0] lv338 = R.call_tir(cls.NT_matmul7, (rms_norm23, model_layers_11_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split11: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv338, indices_or_sections=2, axis=-1) split_011: R.Tensor((1, seq_len, 16384), dtype="float16") = split11[0] split_111: R.Tensor((1, seq_len, 16384), dtype="float16") = split11[1] gelu11: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_011) mul12: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu11, split_111) lv339 = R.call_tir(cls.NT_matmul8, (mul12, model_layers_11_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv334_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv339, lv333_1, model_layers_12_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv335_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv334_1[1] rms_norm24: R.Tensor((1, seq_len, 2048), dtype="float16") = lv334_1[0] lv340 = R.call_tir(cls.NT_matmul5, (rms_norm24, model_layers_12_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape48: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv340, R.shape([1, seq_len, 10, 256])) reshape49: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape48, R.shape([seq_len, 10, 256])) lv12 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape49), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape50: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv12, R.shape([1, seq_len, 8, 256])) reshape51: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape50, R.shape([1, seq_len, 2048])) lv341 = R.call_tir(cls.NT_matmul6, (reshape51, model_layers_12_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv336_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv341, lv335_1, model_layers_12_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv337_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv336_1[1] rms_norm25: R.Tensor((1, seq_len, 2048), dtype="float16") = lv336_1[0] lv342 = R.call_tir(cls.NT_matmul7, (rms_norm25, model_layers_12_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split12: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv342, indices_or_sections=2, axis=-1) split_012: R.Tensor((1, seq_len, 16384), dtype="float16") = split12[0] split_112: R.Tensor((1, seq_len, 16384), dtype="float16") = split12[1] gelu12: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_012) mul13: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu12, split_112) lv343 = R.call_tir(cls.NT_matmul8, (mul13, model_layers_12_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv338_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv343, lv337_1, model_layers_13_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv339_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv338_1[1] rms_norm26: R.Tensor((1, seq_len, 2048), dtype="float16") = lv338_1[0] lv344 = R.call_tir(cls.NT_matmul5, (rms_norm26, model_layers_13_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape52: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv344, R.shape([1, seq_len, 10, 256])) reshape53: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape52, R.shape([seq_len, 10, 256])) lv13 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape53), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape54: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv13, R.shape([1, seq_len, 8, 256])) reshape55: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape54, R.shape([1, seq_len, 2048])) lv345 = R.call_tir(cls.NT_matmul6, (reshape55, model_layers_13_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv340_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv345, lv339_1, model_layers_13_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv341_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv340_1[1] rms_norm27: R.Tensor((1, seq_len, 2048), dtype="float16") = lv340_1[0] lv346 = R.call_tir(cls.NT_matmul7, (rms_norm27, model_layers_13_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split13: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv346, indices_or_sections=2, axis=-1) split_013: R.Tensor((1, seq_len, 16384), dtype="float16") = split13[0] split_113: R.Tensor((1, seq_len, 16384), dtype="float16") = split13[1] gelu13: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_013) mul14: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu13, split_113) lv347 = R.call_tir(cls.NT_matmul8, (mul14, model_layers_13_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv342_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv347, lv341_1, model_layers_14_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv343_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv342_1[1] rms_norm28: R.Tensor((1, seq_len, 2048), dtype="float16") = lv342_1[0] lv348 = R.call_tir(cls.NT_matmul5, (rms_norm28, model_layers_14_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape56: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv348, R.shape([1, seq_len, 10, 256])) reshape57: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape56, R.shape([seq_len, 10, 256])) lv14 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape57), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape58: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv14, R.shape([1, seq_len, 8, 256])) reshape59: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape58, R.shape([1, seq_len, 2048])) lv349 = R.call_tir(cls.NT_matmul6, (reshape59, model_layers_14_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv344_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv349, lv343_1, model_layers_14_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv345_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv344_1[1] rms_norm29: R.Tensor((1, seq_len, 2048), dtype="float16") = lv344_1[0] lv350 = R.call_tir(cls.NT_matmul7, (rms_norm29, model_layers_14_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split14: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv350, indices_or_sections=2, axis=-1) split_014: R.Tensor((1, seq_len, 16384), dtype="float16") = split14[0] split_114: R.Tensor((1, seq_len, 16384), dtype="float16") = split14[1] gelu14: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_014) mul15: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu14, split_114) lv351 = R.call_tir(cls.NT_matmul8, (mul15, model_layers_14_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv346_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv351, lv345_1, model_layers_15_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv347_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv346_1[1] rms_norm30: R.Tensor((1, seq_len, 2048), dtype="float16") = lv346_1[0] lv352 = R.call_tir(cls.NT_matmul5, (rms_norm30, model_layers_15_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape60: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv352, R.shape([1, seq_len, 10, 256])) reshape61: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape60, R.shape([seq_len, 10, 256])) lv15 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape61), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape62: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv15, R.shape([1, seq_len, 8, 256])) reshape63: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape62, R.shape([1, seq_len, 2048])) lv353 = R.call_tir(cls.NT_matmul6, (reshape63, model_layers_15_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv348_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv353, lv347_1, model_layers_15_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv349_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv348_1[1] rms_norm31: R.Tensor((1, seq_len, 2048), dtype="float16") = lv348_1[0] lv354 = R.call_tir(cls.NT_matmul7, (rms_norm31, model_layers_15_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split15: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv354, indices_or_sections=2, axis=-1) split_015: R.Tensor((1, seq_len, 16384), dtype="float16") = split15[0] split_115: R.Tensor((1, seq_len, 16384), dtype="float16") = split15[1] gelu15: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_015) mul16: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu15, split_115) lv355 = R.call_tir(cls.NT_matmul8, (mul16, model_layers_15_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv350_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv355, lv349_1, model_layers_16_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv351_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv350_1[1] rms_norm32: R.Tensor((1, seq_len, 2048), dtype="float16") = lv350_1[0] lv356 = R.call_tir(cls.NT_matmul5, (rms_norm32, model_layers_16_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape64: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv356, R.shape([1, seq_len, 10, 256])) reshape65: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape64, R.shape([seq_len, 10, 256])) lv16 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape65), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape66: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv16, R.shape([1, seq_len, 8, 256])) reshape67: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape66, R.shape([1, seq_len, 2048])) lv357 = R.call_tir(cls.NT_matmul6, (reshape67, model_layers_16_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv352_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv357, lv351_1, model_layers_16_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv353_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv352_1[1] rms_norm33: R.Tensor((1, seq_len, 2048), dtype="float16") = lv352_1[0] lv358 = R.call_tir(cls.NT_matmul7, (rms_norm33, model_layers_16_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split16: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv358, indices_or_sections=2, axis=-1) split_016: R.Tensor((1, seq_len, 16384), dtype="float16") = split16[0] split_116: R.Tensor((1, seq_len, 16384), dtype="float16") = split16[1] gelu16: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_016) mul17: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu16, split_116) lv359 = R.call_tir(cls.NT_matmul8, (mul17, model_layers_16_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv354_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv359, lv353_1, model_layers_17_input_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv355_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv354_1[1] rms_norm34: R.Tensor((1, seq_len, 2048), dtype="float16") = lv354_1[0] lv360 = R.call_tir(cls.NT_matmul5, (rms_norm34, model_layers_17_self_attn_qkv_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2560), dtype="float16")) reshape68: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.reshape(lv360, R.shape([1, seq_len, 10, 256])) reshape69: R.Tensor((seq_len, 10, 256), dtype="float16") = R.reshape(reshape68, R.shape([seq_len, 10, 256])) lv17 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape69), out_sinfo=R.Tensor((seq_len, 8, 256), dtype="float16")) reshape70: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.reshape(lv17, R.shape([1, seq_len, 8, 256])) reshape71: R.Tensor((1, seq_len, 2048), dtype="float16") = R.reshape(reshape70, R.shape([1, seq_len, 2048])) lv361 = R.call_tir(cls.NT_matmul6, (reshape71, model_layers_17_self_attn_o_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv356_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv361, lv355_1, model_layers_17_post_attention_layernorm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) lv357_1: R.Tensor((1, seq_len, 2048), dtype="float16") = lv356_1[1] rms_norm35: R.Tensor((1, seq_len, 2048), dtype="float16") = lv356_1[0] lv362 = R.call_tir(cls.NT_matmul7, (rms_norm35, model_layers_17_mlp_gate_up_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 32768), dtype="float16")) split17: R.Tuple(R.Tensor((1, seq_len, 16384), dtype="float16"), R.Tensor((1, seq_len, 16384), dtype="float16")) = R.split(lv362, indices_or_sections=2, axis=-1) split_017: R.Tensor((1, seq_len, 16384), dtype="float16") = split17[0] split_117: R.Tensor((1, seq_len, 16384), dtype="float16") = split17[1] gelu17: R.Tensor((1, seq_len, 16384), dtype="float16") = R.nn.gelu_tanh(split_017) mul18: R.Tensor((1, seq_len, 16384), dtype="float16") = R.multiply(gelu17, split_117) lv363 = R.call_tir(cls.NT_matmul8, (mul18, model_layers_17_mlp_down_proj_weight1), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float16")) lv358_1 = R.call_tir(cls.fuse_add_norm_prefill, (lv363, lv357_1, model_norm_weight1), out_sinfo=[R.Tensor((1, seq_len, 2048), dtype="float16"), R.Tensor((1, seq_len, 2048), dtype="float16")]) rms_norm36: R.Tensor((1, seq_len, 2048), dtype="float16") = lv358_1[0] lv18 = R.call_tir(cls.index, (rms_norm36,), out_sinfo=R.Tensor((1, 1, 2048), dtype="float16")) lv364 = R.call_tir(cls.NT_matmul14, (lv18, model_embed_tokens_weight1), out_sinfo=R.Tensor((1, 1, vocab_size), dtype="float32")) gv1: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = lv364, paged_kv_cache R.output(gv1) return gv1 @R.function def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64(is_size_var=True) vocab_size = T.int64(is_size_var=True) R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module with R.dataflow(): lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", logits, R.shape([batch_size, vocab_size]), sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) lv1 = R.call_tir(cls.chunk_lse, (lv, temperature), out_sinfo=[R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32"), R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32")]) lv2: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[0] lv3: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[1] lv4 = R.call_tir(cls.softmax_with_chunked_sum, (lv, temperature, lv2, lv3), out_sinfo=R.Tensor((batch_size, vocab_size), dtype="float32")) gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", lv4, R.shape([batch_size, 1, vocab_size]), sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) R.output(gv) return gv