pythia-70m-q4f16-1 / ir_dump /debug-phase1.py
Amai
Add weights
7122674
Raw History Blame Contribute Delete
410 kB
# from tvm.script import ir as I # from tvm.script import tir as T # from tvm.script import relax as R @I.ir_module class Module: I.module_attrs({"system_lib_prefix": "gpt_neox_q4f16_1_"}) @T.prim_func(private=True) def NT_matmul(var_layer_norm39: T.handle, lv95: T.Buffer((T.int64(1536), T.int64(512)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() layer_norm39 = T.match_buffer(var_layer_norm39, (batch_size, T.int64(1), T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(1536)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(1536), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm39[v_i0, v_i1, v_k], lv95[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm39[v_i0, v_i1, v_k] * lv95[v_i2, v_k] @T.prim_func(private=True) def NT_matmul1(var_reshape75: T.handle, lv97: T.Buffer((T.int64(512), T.int64(512)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() reshape75 = T.match_buffer(var_reshape75, (batch_size, T.int64(1), T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(512)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(512), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(reshape75[v_i0, v_i1, v_k], lv97[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape75[v_i0, v_i1, v_k] * lv97[v_i2, v_k] @T.prim_func(private=True) def NT_matmul10(layer_norm13: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), lv33: T.Buffer((T.int64(1536), T.int64(512)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(1536)), "float16")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(1536), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm13[v_i0, v_i1, v_k], lv33[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm13[v_i0, v_i1, v_k] * lv33[v_i2, v_k] @T.prim_func(private=True) def NT_matmul11(reshape27: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), lv35: T.Buffer((T.int64(512), T.int64(512)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(512), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(reshape27[v_i0, v_i1, v_k], lv35[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape27[v_i0, v_i1, v_k] * lv35[v_i2, v_k] @T.prim_func(private=True) def NT_matmul12(layer_norm14: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), lv36: T.Buffer((T.int64(2048), T.int64(512)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float32")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(2048), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm14[v_i0, v_i1, v_k], lv36[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm14[v_i0, v_i1, v_k]) * T.Cast("float32", lv36[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul13(astype13: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16"), lv37: T.Buffer((T.int64(512), T.int64(2048)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float32")): T.func_attr({"tir.noalias": T.bool(True)}) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(512), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(astype13[v_i0, v_i1, v_k], lv37[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", astype13[v_i0, v_i1, v_k]) * T.Cast("float32", lv37[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul14(layer_norm25: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), var_lv63: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) vocab_size = T.int64() lv63 = T.match_buffer(var_lv63, (vocab_size, T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), T.int64(1), vocab_size), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), vocab_size, T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm25[v_i0, v_i1, v_k], lv63[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm25[v_i0, v_i1, v_k] * lv63[v_i2, v_k] @T.prim_func(private=True) def NT_matmul2(var_layer_norm40: T.handle, lv98: T.Buffer((T.int64(2048), T.int64(512)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() layer_norm40 = T.match_buffer(var_layer_norm40, (batch_size, T.int64(1), T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(2048))) # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(2048), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm40[v_i0, v_i1, v_k], lv98[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm40[v_i0, v_i1, v_k]) * T.Cast("float32", lv98[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul3(var_astype39: T.handle, lv99: T.Buffer((T.int64(512), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() astype39 = T.match_buffer(var_astype39, (batch_size, T.int64(1), T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(512))) # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(512), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(astype39[v_i0, v_i1, v_k], lv99[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", astype39[v_i0, v_i1, v_k]) * T.Cast("float32", lv99[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul4(var_layer_norm51: T.handle, var_lv125: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() layer_norm51 = T.match_buffer(var_layer_norm51, (batch_size, T.int64(1), T.int64(512)), "float16") vocab_size = T.int64() lv125 = T.match_buffer(var_lv125, (vocab_size, T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), vocab_size), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(batch_size, T.int64(1), vocab_size, T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm51[v_i0, v_i1, v_k], lv125[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm51[v_i0, v_i1, v_k] * lv125[v_i2, v_k] @T.prim_func(private=True) def NT_matmul5(var_layer_norm26: T.handle, lv64: T.Buffer((T.int64(1536), T.int64(512)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() layer_norm26 = T.match_buffer(var_layer_norm26, (T.int64(1), seq_len, T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(1536)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(1536), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm26[v_i0, v_i1, v_k], lv64[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm26[v_i0, v_i1, v_k] * lv64[v_i2, v_k] @T.prim_func(private=True) def NT_matmul6(var_reshape51: T.handle, lv66: T.Buffer((T.int64(512), T.int64(512)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() reshape51 = T.match_buffer(var_reshape51, (T.int64(1), seq_len, T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(512)), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(512), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(reshape51[v_i0, v_i1, v_k], lv66[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape51[v_i0, v_i1, v_k] * lv66[v_i2, v_k] @T.prim_func(private=True) def NT_matmul7(var_layer_norm27: T.handle, lv67: T.Buffer((T.int64(2048), T.int64(512)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() layer_norm27 = T.match_buffer(var_layer_norm27, (T.int64(1), seq_len, T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(2048))) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(2048), T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(layer_norm27[v_i0, v_i1, v_k], lv67[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm27[v_i0, v_i1, v_k]) * T.Cast("float32", lv67[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul8(var_astype26: T.handle, lv68: T.Buffer((T.int64(512), T.int64(2048)), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) seq_len = T.int64() astype26 = T.match_buffer(var_astype26, (T.int64(1), seq_len, T.int64(2048)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), seq_len, T.int64(512))) # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(512), T.int64(2048)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(astype26[v_i0, v_i1, v_k], lv68[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", astype26[v_i0, v_i1, v_k]) * T.Cast("float32", lv68[v_i2, v_k]) @T.prim_func(private=True) def NT_matmul9(var_take1: T.handle, var_lv94: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.noalias": T.bool(True)}) batch_size = T.int64() take1 = T.match_buffer(var_take1, (T.int64(1), batch_size, T.int64(512)), "float16") vocab_size = T.int64() lv94 = T.match_buffer(var_lv94, (vocab_size, T.int64(512)), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), batch_size, vocab_size), "float16") # with T.block("root"): for i0, i1, i2, k in T.grid(T.int64(1), batch_size, vocab_size, T.int64(512)): with T.block("NT_matmul"): v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k]) T.reads(take1[v_i0, v_i1, v_k], lv94[v_i2, v_k]) T.writes(NT_matmul[v_i0, v_i1, v_i2]) with T.init(): NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0) NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + take1[v_i0, v_i1, v_k] * lv94[v_i2, v_k] @T.prim_func def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") # with T.block("root"): for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) T.writes(logits[seq_ids[vs], vv]) logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) @T.prim_func def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") logit_bias = T.match_buffer(var_logit_bias, (num_token,)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) T.writes(logits[pos2seq_id[vp], token_ids[vp]]) logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] @T.prim_func def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") penalties = T.match_buffer(var_penalties, (num_seq, 3)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) @T.prim_func def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 64), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 64), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.18033688011112042) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"): for ty in T.thread_binding(1, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): for tz in T.thread_binding(16, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68]) T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared") md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((1,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = fused_by_bz % 8 bz: T.int32 = fused_by_bz // 8 batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 15) // 16): tile_start_s: T.int32 = tz + ty tile_start_g: T.int32 = iterator * 16 + tz + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = row_g page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(1): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if iterator * 16 + tz + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(1): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(1): for vec in T.vectorized(4): V_local[vec] = V_smem[tz + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] md_allreduce[tz, ty, 0] = st_m[0] md_allreduce[tz, ty, 1] = st_d[0] T.tvm_storage_sync("shared") st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for j in range(16): m_prev[0] = st_m[0] d_prev[0] = st_d[0] other_m[0] = md_allreduce[j, ty, 0] other_d[0] = md_allreduce[j, ty, 1] for vec in T.vectorized(4): other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] st_m[0] = T.max(st_m[0], other_m[0]) st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) for vec in T.vectorized(4): O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 64), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 64), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.18033688011112042) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"): for ty in T.thread_binding(1, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): for tz in T.thread_binding(16, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68]) T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared") md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((1,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = fused_by_bz % 8 bz: T.int32 = fused_by_bz // 8 batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 15) // 16): tile_start_s: T.int32 = tz + ty tile_start_g: T.int32 = iterator * 16 + tz + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(1): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if iterator * 16 + tz + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(1): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(1): for vec in T.vectorized(4): V_local[vec] = V_smem[tz + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec] md_allreduce[tz, ty, 0] = st_m[0] md_allreduce[tz, ty, 1] = st_d[0] T.tvm_storage_sync("shared") st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for j in range(16): m_prev[0] = st_m[0] d_prev[0] = st_d[0] other_m[0] = md_allreduce[j, ty, 0] other_d[0] = md_allreduce[j, ty, 1] for vec in T.vectorized(4): other_o[vec] = O_allreduce[j, ty, tx * 4 + vec] st_m[0] = T.max(st_m[0], other_m[0]) st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0]) exp_mprev[0] = T.exp2(m_prev[0] - st_m[0]) exp_otherm[0] = T.exp2(other_m[0] - st_m[0]) for vec in T.vectorized(4): O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 16), scope="shared") S_local = T.alloc_buffer((32, 16), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64]) T.writes(S_local[0:32, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((16, 64), "float16", scope="shared") V_smem = T.alloc_buffer((16, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 16), scope="shared") S_local = T.alloc_buffer((32, 16), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(2): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64]) T.writes(S_local[0:32, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 8, 64), "float16") v = T.match_buffer(var_v, (kv_len, 8, 64), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(8, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((64, 32), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] q_indptr_val: T.int32 = q_indptr[b_idx] LH_start: T.int32 = tile_id[0] * 32 kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_0, lj_0_0 in T.grid(1, 2): for lj_1 in T.vectorized(8): with T.block("Q_load"): i = T.axis.spatial(32, li_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 4) j = T.axis.spatial(64, lj_0_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 4 * 8 + lj_1) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(1, 2): for ly_1 in T.vectorized(8): with T.block("K_load"): i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4) j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1) T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 32:j - 32 + 65]) T.writes(K_smem[j, i]) cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: freq = T.float32() K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[L_kv_base + cur_L, by, j + 32] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[L_kv_base + cur_L, by, j]) else: K_smem[j, i] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(1, 2): for ly_1 in T.vectorized(8): with T.block("V_load"): i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4) j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: V_smem[i, j] = v[L_kv_base + cur_L, by, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:64, 0:32]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(2): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_0_init * 4 + lj_1_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0 in range(4): for li_1 in T.unroll(2): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): for lk_1 in range(16): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) k_1 = T.axis.reduce(64, lk_0 * 16 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(2): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(4): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_0_init * 4 + lj_1_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1 in T.grid(2, 16): for li_1 in T.unroll(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) k_1 = T.axis.reduce(32, lk_0 * 16 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 8 @T.prim_func def batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 64), "float16") q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 8, 64), "float16") v = T.match_buffer(var_v, (kv_len, 8, 64), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) tree_size = T.int32(is_size_var=True) mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((32, 64), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("KV_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_base + L_kv_start + i if L_kv_start + i < kv_chunk_len[0]: freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[cur_L, by, j + 32] * T.float16(-1.0), k[cur_L, by, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[cur_L, by, j]) V_smem[i, j] = v[cur_L, by, j] else: K_smem[i, j] = T.float16(0.0) V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) k_1 = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int64(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) # with T.block("root"): A_pad = T.alloc_buffer((batch_size, num_chunks, T.int64(4096))) temp_max = T.alloc_buffer((batch_size, num_chunks)) temp_sum = T.alloc_buffer((batch_size, num_chunks)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("pad"): v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2]) T.writes(A_pad[v0, v1, v2]) A_pad[v0, v1, v2] = T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * T.int64(4096) + v2] / temperature[v0], A[v0, v1 * T.int64(4096) + v2]), T.float32(-340282346638528859811704183484516925440.0)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("max"): v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) T.reads(A_pad[v0, v1, v2]) T.writes(temp_max[v0, v1]) with T.init(): temp_max[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) temp_max[v0, v1] = T.max(temp_max[v0, v1], A_pad[v0, v1, v2]) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)): with T.block("sum_exp"): v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2]) T.reads(temperature[v0], A_pad[v0, v1, v2], temp_max[v0, v1]) T.writes(temp_sum[v0, v1]) with T.init(): temp_sum[v0, v1] = T.float32(0.0) temp_sum[v0, v1] = temp_sum[v0, v1] + T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A_pad[v0, v1, v2] - temp_max[v0, v1]), T.Cast("float32", A_pad[v0, v1, v2] == temp_max[v0, v1])), T.float32(0.0)) for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(1)): with T.block("log"): v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2]) T.reads(temperature[v0], temp_sum[v0, v1], temp_max[v0, v1]) T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum[v0, v1]), temp_sum[v0, v1]) chunked_max[v0, v1] = temp_max[v0, v1] @T.prim_func def compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 8, 16, 64), "float16", offset_factor=1) copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) total_copy_length = T.int32() copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) with T.block("root"): T.reads() T.writes() for bhd_o in T.thread_binding(batch_size * 2, thread="blockIdx.x"): for bhd_i in T.thread_binding(256, thread="threadIdx.x"): b: T.int32 = (bhd_o * 256 + bhd_i) // 512 h: T.int32 = (bhd_o * 256 + bhd_i) // 64 % 8 d: T.int32 = (bhd_o * 256 + bhd_i) % 64 if bhd_o * 256 + bhd_i < batch_size * 8 * 64: for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] @T.prim_func def copy_single_page(var_pages: T.handle, src_page_id: T.int64, tgt_page_id: T.int64, copy_length: T.int64): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages, page_size = T.int32(), T.int64() pages = T.match_buffer(var_pages, (num_pages, 2, 8, page_size, 64), "float16", offset_factor=1) # with T.block("root"): for b in T.thread_binding(copy_length * T.int64(2), thread="blockIdx.x"): for t in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy"): vh = T.axis.spatial(8, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) // (copy_length * T.int64(64)))) vp = T.axis.spatial(copy_length, (b * T.int64(256) + T.Cast("int64", t)) % (copy_length * T.int64(64)) // T.int64(64)) vd = T.axis.spatial(64, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) % T.int64(64))) T.where(b * T.int64(256) + T.Cast("int64", t) < copy_length * T.int64(8) * T.int64(64)) T.reads(pages[src_page_id, 0:2, vh, vp, vd]) T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] @T.prim_func(private=True) def dequantize(var_gpt_neox_embed_in_q_weight: T.handle, var_gpt_neox_embed_in_q_scale: T.handle, var_dequantize: T.handle): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) vocab_size = T.int64() gpt_neox_embed_in_q_weight = T.match_buffer(var_gpt_neox_embed_in_q_weight, (vocab_size, T.int64(64)), "uint32") gpt_neox_embed_in_q_scale = T.match_buffer(var_gpt_neox_embed_in_q_scale, (vocab_size, T.int64(16)), "float16") dequantize = T.match_buffer(var_dequantize, (vocab_size, T.int64(512)), "float16") # with T.block("root"): compute = T.alloc_buffer((vocab_size, T.int64(512)), "float16") for i0, i1 in T.grid(vocab_size, T.int64(512)): with T.block("compute"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(gpt_neox_embed_in_q_weight[v_i0, v_i1 // T.int64(8)]) T.writes(compute[v_i0, v_i1]) compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_embed_in_q_weight[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15))) for i0, i1 in T.grid(vocab_size, T.int64(512)): with T.block("dequantize"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(compute[v_i0, v_i1], gpt_neox_embed_in_q_scale[v_i0, v_i1 // T.int64(32)]) T.writes(dequantize[v_i0, v_i1]) dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_embed_in_q_scale[v_i0, v_i1 // T.int64(32)] @T.prim_func(private=True) def dequantize1(gpt_neox_layers_0_attention_query_key_value_q_weight1: T.Buffer((T.int64(1536), T.int64(64)), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale1: T.Buffer((T.int64(1536), T.int64(16)), "float16"), dequantize: T.Buffer((T.int64(1536), T.int64(512)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) # with T.block("root"): compute = T.alloc_buffer((T.int64(1536), T.int64(512)), "float16") for i0, i1 in T.grid(T.int64(1536), T.int64(512)): with T.block("compute"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight1[v_i0, v_i1 // T.int64(8)]) T.writes(compute[v_i0, v_i1]) compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15))) for i0, i1 in T.grid(T.int64(1536), T.int64(512)): with T.block("dequantize"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_attention_query_key_value_q_scale1[v_i0, v_i1 // T.int64(32)]) T.writes(dequantize[v_i0, v_i1]) dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale1[v_i0, v_i1 // T.int64(32)] @T.prim_func(private=True) def dequantize2(gpt_neox_layers_0_attention_dense_q_weight1: T.Buffer((T.int64(512), T.int64(64)), "uint32"), gpt_neox_layers_0_attention_dense_q_scale1: T.Buffer((T.int64(512), T.int64(16)), "float16"), dequantize: T.Buffer((T.int64(512), T.int64(512)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) # with T.block("root"): compute = T.alloc_buffer((T.int64(512), T.int64(512)), "float16") for i0, i1 in T.grid(T.int64(512), T.int64(512)): with T.block("compute"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(gpt_neox_layers_0_attention_dense_q_weight1[v_i0, v_i1 // T.int64(8)]) T.writes(compute[v_i0, v_i1]) compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15))) for i0, i1 in T.grid(T.int64(512), T.int64(512)): with T.block("dequantize"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_attention_dense_q_scale1[v_i0, v_i1 // T.int64(32)]) T.writes(dequantize[v_i0, v_i1]) dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale1[v_i0, v_i1 // T.int64(32)] @T.prim_func(private=True) def dequantize3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1: T.Buffer((T.int64(2048), T.int64(64)), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1: T.Buffer((T.int64(2048), T.int64(16)), "float16"), dequantize: T.Buffer((T.int64(2048), T.int64(512)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) # with T.block("root"): compute = T.alloc_buffer((T.int64(2048), T.int64(512)), "float16") for i0, i1 in T.grid(T.int64(2048), T.int64(512)): with T.block("compute"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1[v_i0, v_i1 // T.int64(8)]) T.writes(compute[v_i0, v_i1]) compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15))) for i0, i1 in T.grid(T.int64(2048), T.int64(512)): with T.block("dequantize"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1[v_i0, v_i1 // T.int64(32)]) T.writes(dequantize[v_i0, v_i1]) dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1[v_i0, v_i1 // T.int64(32)] @T.prim_func(private=True) def dequantize4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1: T.Buffer((T.int64(512), T.int64(256)), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1: T.Buffer((T.int64(512), T.int64(64)), "float16"), dequantize: T.Buffer((T.int64(512), T.int64(2048)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) # with T.block("root"): compute = T.alloc_buffer((T.int64(512), T.int64(2048)), "float16") for i0, i1 in T.grid(T.int64(512), T.int64(2048)): with T.block("compute"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1[v_i0, v_i1 // T.int64(8)]) T.writes(compute[v_i0, v_i1]) compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15))) for i0, i1 in T.grid(T.int64(512), T.int64(2048)): with T.block("dequantize"): v_i0, v_i1 = T.axis.remap("SS", [i0, i1]) T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1[v_i0, v_i1 // T.int64(32)]) T.writes(dequantize[v_i0, v_i1]) dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1[v_i0, v_i1 // T.int64(32)] @T.prim_func def fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) seq_len = T.int32() qkv = T.match_buffer(var_qkv, (seq_len, 24, 64), "float16") position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) q = T.match_buffer(var_q, (seq_len, 8, 64), "float16") k = T.match_buffer(var_k, (seq_len, 8, 64), "float16") v = T.match_buffer(var_v, (seq_len, 8, 64), "float16") # with T.block("root"): for iters_0, iters_1, iters_2 in T.grid(seq_len, 24, 64): with T.block("llama_fused_rope"): s, h, d = T.axis.remap("SSS", [iters_0, iters_1, iters_2]) T.reads(position_map[s], qkv[s, h, d - 8:d - 8 + 17]) T.writes(q[s, h, d], k[s, h - 8, d], v[s, h - 16, d]) if h < 8: freq = T.float32() q[s, h, d] = T.if_then_else(apply_rope > 0 and d < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 8, qkv[s, h, d + 8] * T.float16(-1.0), qkv[s, h, d - 8]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 16) / T.float32(16.0))}), qkv[s, h, d]) else: if h < 16: freq = T.float32() k[s, h - 8, d] = T.if_then_else(apply_rope > 0 and d < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 8, qkv[s, h, d + 8] * T.float16(-1.0), qkv[s, h, d - 8]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 16) / T.float32(16.0))}), qkv[s, h, d]) else: v[s, h - 16, d] = qkv[s, h, d] @T.prim_func def gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (m, n)) batch_size = T.int32(is_size_var=True) indices = T.match_buffer(var_indices, (batch_size,), "int32") dst = T.match_buffer(var_dst, (batch_size, n)) # with T.block("root"): for b, j in T.grid(batch_size, n): with T.block("gather_2d"): vb, vj = T.axis.remap("SS", [b, j]) T.reads(src[indices[vb], vj], indices[vb]) T.writes(dst[vb, vj]) dst[vb, vj] = src[indices[vb], vj] @T.prim_func(private=True) def index(var_layer_norm12: T.handle, index: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) seq_len = T.int64() layer_norm12 = T.match_buffer(var_layer_norm12, (T.int64(1), seq_len, T.int64(512)), "float16") # with T.block("root"): for i, _, k in T.grid(T.int64(1), T.int64(1), T.int64(512)): with T.block("index"): v_i, v__, v_k = T.axis.remap("SSS", [i, _, k]) T.reads(layer_norm12[v_i, seq_len - T.int64(1), v_k]) T.writes(index[v_i, v__, v_k]) index[v_i, v__, v_k] = layer_norm12[v_i, seq_len - T.int64(1), v_k] @T.prim_func def merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) V = T.match_buffer(v, (N, H, D), "float16") S = T.match_buffer(s, (N, H)) V_other = T.match_buffer(v_other, (N, H, D), "float16") S_other = T.match_buffer(s_other, (N, H)) # with T.block("root"): for bx in T.thread_binding(N, thread="blockIdx.x"): for by in T.thread_binding(1, thread="blockIdx.y"): for ty in T.thread_binding(8, thread="threadIdx.y"): for tx in T.thread_binding(16, thread="threadIdx.x"): with T.block("merge"): T.reads(S[bx, ty + by * 8], S_other[bx, ty + by * 8], V[bx, ty + by * 8, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 8, tx * 4:tx * 4 + 4]) T.writes(V[bx, ty + by * 8, tx * 4:tx * 4 + 4], S[bx, ty + by * 8]) s_val = T.alloc_buffer((1,), scope="local") s_other_val = T.alloc_buffer((1,), scope="local") s_max = T.alloc_buffer((1,), scope="local") scale = T.alloc_buffer((1,), scope="local") other_scale = T.alloc_buffer((1,), scope="local") v_vec = T.alloc_buffer((4,), "float16", scope="local") v_other_vec = T.alloc_buffer((4,), "float16", scope="local") s_val[0] = S[bx, ty + by * 8] s_other_val[0] = S_other[bx, ty + by * 8] s_max[0] = T.max(s_val[0], s_other_val[0]) s_val[0] = T.exp2(s_val[0] - s_max[0]) s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) for vec in T.vectorized(4): v_vec[vec] = V[bx, ty + by * 8, tx * 4 + vec] for vec in T.vectorized(4): v_other_vec[vec] = V_other[bx, ty + by * 8, tx * 4 + vec] for vec in range(4): v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) for vec in T.vectorized(4): V[bx, ty + by * 8, tx * 4 + vec] = v_vec[vec] S[bx, ty + by * 8] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] @T.prim_func def scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (batch_size, n)) indices = T.match_buffer(var_indices, (batch_size,), "int32") m = T.int32(is_size_var=True) dst = T.match_buffer(var_dst, (m, n)) # with T.block("root"): for b, j in T.grid(batch_size, n): with T.block("scatter_2d"): vb, vj = T.axis.remap("SS", [b, j]) T.reads(src[vb, vj], indices[vb]) T.writes(dst[indices[vb], vj]) dst[indices[vb], vj] = src[vb, vj] @T.prim_func def softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int64(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) T.reads(chunked_max[v0, v1]) T.writes(temp_max_shared[v0]) with T.init(): temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1) T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks) T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) T.writes(temp_sum_shared[v0]) with T.init(): temp_sum_shared[v0] = T.float32(0.0) temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) for l2_0 in T.serial(T.int64(16), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): for l2_1 in T.thread_binding(T.int64(8), thread="threadIdx.y"): for l2_2 in T.thread_binding(T.int64(32), thread="threadIdx.x"): with T.block("log_pad"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) v2 = T.axis.spatial(T.int64(4096), l2_0 * T.int64(256) + l2_1 * T.int64(32) + l2_2) T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2], temp_sum_shared[v0], temp_max_shared[v0]) T.writes(softmax[v0, v1 * T.int64(4096) + v2]) if v1 * T.int64(4096) + v2 < vocab_size: softmax[v0, v1 * T.int64(4096) + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * T.int64(4096) + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * T.int64(4096) + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) @T.prim_func def tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int64): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) num_pages, page_size = T.int64(), T.int64(is_size_var=True) pages = T.match_buffer(var_pages, (num_pages, 2, 8, page_size, 64), "float16", offset_factor=1) seqlen = T.int64(is_size_var=True) position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) k_data = T.match_buffer(var_k_data, (6, seqlen, 8, 64), "float16") v_data = T.match_buffer(var_v_data, (6, seqlen, 8, 64), "float16") # with T.block("root"): for p, h, d in T.grid(seqlen, 8, 64): with T.block("copy0"): vp, vh, vd = T.axis.remap("SSS", [p, h, d]) T.reads(position_map[vp], pages[T.Cast("int64", position_map[vp]) // page_size, 0:2, vh, T.Cast("int64", position_map[vp]) % page_size, vd]) T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) position: T.int32 = position_map[vp] k_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 0, vh, T.Cast("int64", position) % page_size, vd] v_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 1, vh, T.Cast("int64", position) % page_size, vd] @T.prim_func def tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)}) num_pages = T.int64() pages = T.match_buffer(var_pages, (num_pages, 2, 8, 16, 64), "float16", offset_factor=1) ntoken = T.int64(is_size_var=True) k_data = T.match_buffer(var_k_data, (ntoken, 8, 64), "float16") v_data = T.match_buffer(var_v_data, (ntoken, 8, 64), "float16") position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) # with T.block("root"): for global_pos, h, f in T.grid(ntoken, 8, 64): if position_map[global_pos] != -1: with T.block("k_transpose_append"): vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] with T.block("v_transpose_append"): vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f]) T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] @T.prim_func def tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 64), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 32, 64), "float16") page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 64), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) total_tree_order_len = T.int32(is_size_var=True) tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) # with T.block("root"): assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(8, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared") K_smem = T.alloc_buffer((32, 64), "float16", scope="shared") V_smem = T.alloc_buffer((32, 64), "float16", scope="shared") S_smem = T.alloc_buffer((32, 32), scope="shared") S_local = T.alloc_buffer((32, 32), scope="local") O_local = T.alloc_buffer((32, 64), scope="local") m_smem = T.alloc_buffer((32,), scope="shared") m_prev_smem = T.alloc_buffer((32,), scope="shared") d_smem = T.alloc_buffer((32,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = q_indptr[1] - q_indptr[0] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx] batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 32 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(4): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64) j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 K_smem[i, j] = pages[page_no, 0, by, page_offset, j] else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(4): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64) j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64]) T.writes(S_local[0:32, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 4): with T.block("S_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8): with T.block("S_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) k = T.axis.reduce(64, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 4): with T.block("S_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = LH_start + row for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 32: row_: T.int32 = LH_start + row if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 32: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64]) T.writes(O_local[0:32, 0:64]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 4): with T.block("O_gemm_init"): i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init) j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4): with T.block("O_gemm_update"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) k = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 4): with T.block("O_store"): i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1) j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 32) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i), by]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) cur_H_qo: T.int32 = by if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @R.function def alloc_embedding_tensor() -> R.Tensor((2048, 512), dtype="float16"): R.func_attr({"relax.memory_plan_dynamic_func_output": True}) gv: R.Tensor((2048, 512), dtype="float16") = R.builtin.alloc_tensor(R.shape([2048, 512]), R.dtype("float16"), R.prim_value(0), R.str("global")) return gv @R.function def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): gpt_neox_layers_0_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[3] gpt_neox_layers_0_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[5] gpt_neox_layers_0_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[8] gpt_neox_layers_0_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[11] gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[14] gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[17] gpt_neox_layers_1_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[19] gpt_neox_layers_1_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[21] gpt_neox_layers_1_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[24] gpt_neox_layers_1_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[27] gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[30] gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[33] gpt_neox_layers_2_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[35] gpt_neox_layers_2_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[37] gpt_neox_layers_2_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[40] gpt_neox_layers_2_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[43] gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[46] gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[49] gpt_neox_layers_3_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[51] gpt_neox_layers_3_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[53] gpt_neox_layers_3_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[56] gpt_neox_layers_3_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[59] gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[62] gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[65] gpt_neox_layers_4_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[67] gpt_neox_layers_4_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[69] gpt_neox_layers_4_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[72] gpt_neox_layers_4_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[75] gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[78] gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[81] gpt_neox_layers_5_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[83] gpt_neox_layers_5_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[85] gpt_neox_layers_5_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[88] gpt_neox_layers_5_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[91] gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[94] gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[97] gpt_neox_final_layer_norm_weight4: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias4: R.Tensor((512,), dtype="float16") = packed_params[99] embed_out_q_weight4: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale4: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] layer_norm39: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight4, gpt_neox_layers_0_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv95 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight4, gpt_neox_layers_0_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv = R.call_tir(cls.NT_matmul, (layer_norm39, lv95), out_sinfo=R.Tensor((batch_size, 1, 1536), dtype="float16")) add108: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(lv, gpt_neox_layers_0_attention_query_key_value_bias4) reshape72: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add108, R.shape([batch_size, 1, 24, 64])) reshape73: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape72, R.shape([batch_size, 24, 64])) lv96 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape73), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16")) reshape74: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv96, R.shape([batch_size, 1, 8, 64])) reshape75: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape74, R.shape([batch_size, 1, 512])) lv97 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight4, gpt_neox_layers_0_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv1 = R.call_tir(cls.NT_matmul1, (reshape75, lv97), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float16")) add109: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(lv1, gpt_neox_layers_0_attention_dense_bias4) layer_norm40: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight4, gpt_neox_layers_0_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv98 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv2 = R.call_tir(cls.NT_matmul2, (layer_norm40, lv98), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float32")) add110: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(lv2, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4) gelu18: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add110) astype39: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu18, dtype="float16") lv99 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv3 = R.call_tir(cls.NT_matmul3, (astype39, lv99), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float32")) add111: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(lv3, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4) astype40: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add111, dtype="float16") add112: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype40, add109) add113: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add112, input_embeds) layer_norm41: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add113, gpt_neox_layers_1_input_layernorm_weight4, gpt_neox_layers_1_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv100 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight4, gpt_neox_layers_1_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv4 = R.call_tir(cls.NT_matmul, (layer_norm41, lv100), out_sinfo=R.Tensor((batch_size, 1, 1536), dtype="float16")) add114: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(lv4, gpt_neox_layers_1_attention_query_key_value_bias4) reshape76: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add114, R.shape([batch_size, 1, 24, 64])) reshape77: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape76, R.shape([batch_size, 24, 64])) lv101 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape77), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16")) reshape78: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv101, R.shape([batch_size, 1, 8, 64])) reshape79: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape78, R.shape([batch_size, 1, 512])) lv102 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight4, gpt_neox_layers_1_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv5 = R.call_tir(cls.NT_matmul1, (reshape79, lv102), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float16")) add115: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(lv5, gpt_neox_layers_1_attention_dense_bias4) layer_norm42: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add113, gpt_neox_layers_1_post_attention_layernorm_weight4, gpt_neox_layers_1_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv103 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv6 = R.call_tir(cls.NT_matmul2, (layer_norm42, lv103), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float32")) add116: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(lv6, gpt_neox_layers_1_mlp_dense_h_to_4h_bias4) gelu19: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add116) astype41: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu19, dtype="float16") lv104 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv7 = R.call_tir(cls.NT_matmul3, (astype41, lv104), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float32")) add117: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(lv7, gpt_neox_layers_1_mlp_dense_4h_to_h_bias4) astype42: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add117, dtype="float16") add118: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype42, add115) add119: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add118, add113) layer_norm43: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add119, gpt_neox_layers_2_input_layernorm_weight4, gpt_neox_layers_2_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv105 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight4, gpt_neox_layers_2_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv8 = R.call_tir(cls.NT_matmul, (layer_norm43, lv105), out_sinfo=R.Tensor((batch_size, 1, 1536), dtype="float16")) add120: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(lv8, gpt_neox_layers_2_attention_query_key_value_bias4) reshape80: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add120, R.shape([batch_size, 1, 24, 64])) reshape81: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape80, R.shape([batch_size, 24, 64])) lv106 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape81), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16")) reshape82: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv106, R.shape([batch_size, 1, 8, 64])) reshape83: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape82, R.shape([batch_size, 1, 512])) lv107 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight4, gpt_neox_layers_2_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv9 = R.call_tir(cls.NT_matmul1, (reshape83, lv107), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float16")) add121: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(lv9, gpt_neox_layers_2_attention_dense_bias4) layer_norm44: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add119, gpt_neox_layers_2_post_attention_layernorm_weight4, gpt_neox_layers_2_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv108 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv10 = R.call_tir(cls.NT_matmul2, (layer_norm44, lv108), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float32")) add122: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(lv10, gpt_neox_layers_2_mlp_dense_h_to_4h_bias4) gelu20: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add122) astype43: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu20, dtype="float16") lv109 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv11 = R.call_tir(cls.NT_matmul3, (astype43, lv109), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float32")) add123: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(lv11, gpt_neox_layers_2_mlp_dense_4h_to_h_bias4) astype44: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add123, dtype="float16") add124: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype44, add121) add125: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add124, add119) layer_norm45: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add125, gpt_neox_layers_3_input_layernorm_weight4, gpt_neox_layers_3_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv110 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight4, gpt_neox_layers_3_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv12 = R.call_tir(cls.NT_matmul, (layer_norm45, lv110), out_sinfo=R.Tensor((batch_size, 1, 1536), dtype="float16")) add126: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(lv12, gpt_neox_layers_3_attention_query_key_value_bias4) reshape84: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add126, R.shape([batch_size, 1, 24, 64])) reshape85: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape84, R.shape([batch_size, 24, 64])) lv111 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape85), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16")) reshape86: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv111, R.shape([batch_size, 1, 8, 64])) reshape87: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape86, R.shape([batch_size, 1, 512])) lv112 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight4, gpt_neox_layers_3_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv13 = R.call_tir(cls.NT_matmul1, (reshape87, lv112), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float16")) add127: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(lv13, gpt_neox_layers_3_attention_dense_bias4) layer_norm46: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add125, gpt_neox_layers_3_post_attention_layernorm_weight4, gpt_neox_layers_3_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv113 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv14 = R.call_tir(cls.NT_matmul2, (layer_norm46, lv113), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float32")) add128: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(lv14, gpt_neox_layers_3_mlp_dense_h_to_4h_bias4) gelu21: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add128) astype45: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu21, dtype="float16") lv114 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv15 = R.call_tir(cls.NT_matmul3, (astype45, lv114), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float32")) add129: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(lv15, gpt_neox_layers_3_mlp_dense_4h_to_h_bias4) astype46: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add129, dtype="float16") add130: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype46, add127) add131: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add130, add125) layer_norm47: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add131, gpt_neox_layers_4_input_layernorm_weight4, gpt_neox_layers_4_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv115 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight4, gpt_neox_layers_4_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv16 = R.call_tir(cls.NT_matmul, (layer_norm47, lv115), out_sinfo=R.Tensor((batch_size, 1, 1536), dtype="float16")) add132: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(lv16, gpt_neox_layers_4_attention_query_key_value_bias4) reshape88: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add132, R.shape([batch_size, 1, 24, 64])) reshape89: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape88, R.shape([batch_size, 24, 64])) lv116 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape89), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16")) reshape90: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv116, R.shape([batch_size, 1, 8, 64])) reshape91: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape90, R.shape([batch_size, 1, 512])) lv117 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight4, gpt_neox_layers_4_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv17 = R.call_tir(cls.NT_matmul1, (reshape91, lv117), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float16")) add133: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(lv17, gpt_neox_layers_4_attention_dense_bias4) layer_norm48: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add131, gpt_neox_layers_4_post_attention_layernorm_weight4, gpt_neox_layers_4_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv118 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv18 = R.call_tir(cls.NT_matmul2, (layer_norm48, lv118), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float32")) add134: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(lv18, gpt_neox_layers_4_mlp_dense_h_to_4h_bias4) gelu22: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add134) astype47: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu22, dtype="float16") lv119 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv19 = R.call_tir(cls.NT_matmul3, (astype47, lv119), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float32")) add135: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(lv19, gpt_neox_layers_4_mlp_dense_4h_to_h_bias4) astype48: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add135, dtype="float16") add136: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype48, add133) add137: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add136, add131) layer_norm49: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add137, gpt_neox_layers_5_input_layernorm_weight4, gpt_neox_layers_5_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv120 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight4, gpt_neox_layers_5_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv20 = R.call_tir(cls.NT_matmul, (layer_norm49, lv120), out_sinfo=R.Tensor((batch_size, 1, 1536), dtype="float16")) add138: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(lv20, gpt_neox_layers_5_attention_query_key_value_bias4) reshape92: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add138, R.shape([batch_size, 1, 24, 64])) reshape93: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape92, R.shape([batch_size, 24, 64])) lv121 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape93), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16")) reshape94: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv121, R.shape([batch_size, 1, 8, 64])) reshape95: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape94, R.shape([batch_size, 1, 512])) lv122 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight4, gpt_neox_layers_5_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv21 = R.call_tir(cls.NT_matmul1, (reshape95, lv122), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float16")) add139: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(lv21, gpt_neox_layers_5_attention_dense_bias4) layer_norm50: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add137, gpt_neox_layers_5_post_attention_layernorm_weight4, gpt_neox_layers_5_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv123 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv22 = R.call_tir(cls.NT_matmul2, (layer_norm50, lv123), out_sinfo=R.Tensor((batch_size, 1, 2048), dtype="float32")) add140: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(lv22, gpt_neox_layers_5_mlp_dense_h_to_4h_bias4) gelu23: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add140) astype49: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu23, dtype="float16") lv124 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv23 = R.call_tir(cls.NT_matmul3, (astype49, lv124), out_sinfo=R.Tensor((batch_size, 1, 512), dtype="float32")) add141: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(lv23, gpt_neox_layers_5_mlp_dense_4h_to_h_bias4) astype50: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add141, dtype="float16") add142: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype50, add139) add143: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add142, add137) layer_norm51: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add143, gpt_neox_final_layer_norm_weight4, gpt_neox_final_layer_norm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv125 = R.call_tir(cls.dequantize, (embed_out_q_weight4, embed_out_q_scale4), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16")) lv24 = R.call_tir(cls.NT_matmul4, (layer_norm51, lv125), out_sinfo=R.Tensor((batch_size, 1, vocab_size), dtype="float16")) astype51: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.astype(lv24, dtype="float32") gv4: R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype="float32"), R.Object) = astype51, paged_kv_cache R.output(gv4) return gv4 @R.function def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 512), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): gpt_neox_layers_0_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[3] gpt_neox_layers_0_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[5] gpt_neox_layers_0_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[8] gpt_neox_layers_0_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[11] gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[14] gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[17] gpt_neox_layers_1_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[19] gpt_neox_layers_1_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[21] gpt_neox_layers_1_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[24] gpt_neox_layers_1_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[27] gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[30] gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[33] gpt_neox_layers_2_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[35] gpt_neox_layers_2_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[37] gpt_neox_layers_2_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[40] gpt_neox_layers_2_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[43] gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[46] gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[49] gpt_neox_layers_3_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[51] gpt_neox_layers_3_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[53] gpt_neox_layers_3_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[56] gpt_neox_layers_3_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[59] gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[62] gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[65] gpt_neox_layers_4_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[67] gpt_neox_layers_4_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[69] gpt_neox_layers_4_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[72] gpt_neox_layers_4_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[75] gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[78] gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[81] gpt_neox_layers_5_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[83] gpt_neox_layers_5_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[85] gpt_neox_layers_5_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[88] gpt_neox_layers_5_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[91] gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[94] gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[97] gpt_neox_final_layer_norm_weight3: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias3: R.Tensor((512,), dtype="float16") = packed_params[99] embed_out_q_weight3: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale3: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] layer_norm26: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight3, gpt_neox_layers_0_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv64 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight3, gpt_neox_layers_0_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv25 = R.call_tir(cls.NT_matmul5, (layer_norm26, lv64), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add72: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv25, gpt_neox_layers_0_attention_query_key_value_bias3) reshape48: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add72, R.shape([1, seq_len, 24, 64])) reshape49: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape48, R.shape([seq_len, 24, 64])) lv65 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape49), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape50: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv65, R.shape([1, seq_len, 8, 64])) reshape51: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape50, R.shape([1, seq_len, 512])) lv66 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight3, gpt_neox_layers_0_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv26 = R.call_tir(cls.NT_matmul6, (reshape51, lv66), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add73: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv26, gpt_neox_layers_0_attention_dense_bias3) layer_norm27: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight3, gpt_neox_layers_0_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv67 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv27 = R.call_tir(cls.NT_matmul7, (layer_norm27, lv67), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add74: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv27, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3) gelu12: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add74) astype26: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu12, dtype="float16") lv68 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv28 = R.call_tir(cls.NT_matmul8, (astype26, lv68), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add75: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv28, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3) astype27: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add75, dtype="float16") add76: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype27, add73) add77: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add76, input_embeds) layer_norm28: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add77, gpt_neox_layers_1_input_layernorm_weight3, gpt_neox_layers_1_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv69 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight3, gpt_neox_layers_1_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv29 = R.call_tir(cls.NT_matmul5, (layer_norm28, lv69), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add78: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv29, gpt_neox_layers_1_attention_query_key_value_bias3) reshape52: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add78, R.shape([1, seq_len, 24, 64])) reshape53: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape52, R.shape([seq_len, 24, 64])) lv70 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape53), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape54: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv70, R.shape([1, seq_len, 8, 64])) reshape55: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape54, R.shape([1, seq_len, 512])) lv71 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight3, gpt_neox_layers_1_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv30 = R.call_tir(cls.NT_matmul6, (reshape55, lv71), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add79: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv30, gpt_neox_layers_1_attention_dense_bias3) layer_norm29: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add77, gpt_neox_layers_1_post_attention_layernorm_weight3, gpt_neox_layers_1_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv72 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv31 = R.call_tir(cls.NT_matmul7, (layer_norm29, lv72), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add80: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv31, gpt_neox_layers_1_mlp_dense_h_to_4h_bias3) gelu13: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add80) astype28: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu13, dtype="float16") lv73 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv32 = R.call_tir(cls.NT_matmul8, (astype28, lv73), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add81: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv32, gpt_neox_layers_1_mlp_dense_4h_to_h_bias3) astype29: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add81, dtype="float16") add82: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype29, add79) add83: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add82, add77) layer_norm30: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add83, gpt_neox_layers_2_input_layernorm_weight3, gpt_neox_layers_2_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv74 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight3, gpt_neox_layers_2_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv33 = R.call_tir(cls.NT_matmul5, (layer_norm30, lv74), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add84: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv33, gpt_neox_layers_2_attention_query_key_value_bias3) reshape56: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add84, R.shape([1, seq_len, 24, 64])) reshape57: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape56, R.shape([seq_len, 24, 64])) lv75 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape57), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape58: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv75, R.shape([1, seq_len, 8, 64])) reshape59: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape58, R.shape([1, seq_len, 512])) lv76 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight3, gpt_neox_layers_2_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv34 = R.call_tir(cls.NT_matmul6, (reshape59, lv76), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add85: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv34, gpt_neox_layers_2_attention_dense_bias3) layer_norm31: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add83, gpt_neox_layers_2_post_attention_layernorm_weight3, gpt_neox_layers_2_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv77 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv35 = R.call_tir(cls.NT_matmul7, (layer_norm31, lv77), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add86: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv35, gpt_neox_layers_2_mlp_dense_h_to_4h_bias3) gelu14: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add86) astype30: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu14, dtype="float16") lv78 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv36 = R.call_tir(cls.NT_matmul8, (astype30, lv78), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add87: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv36, gpt_neox_layers_2_mlp_dense_4h_to_h_bias3) astype31: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add87, dtype="float16") add88: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype31, add85) add89: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add88, add83) layer_norm32: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add89, gpt_neox_layers_3_input_layernorm_weight3, gpt_neox_layers_3_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv79 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight3, gpt_neox_layers_3_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv37 = R.call_tir(cls.NT_matmul5, (layer_norm32, lv79), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add90: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv37, gpt_neox_layers_3_attention_query_key_value_bias3) reshape60: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add90, R.shape([1, seq_len, 24, 64])) reshape61: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape60, R.shape([seq_len, 24, 64])) lv80 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape61), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape62: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv80, R.shape([1, seq_len, 8, 64])) reshape63: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape62, R.shape([1, seq_len, 512])) lv81 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight3, gpt_neox_layers_3_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv38 = R.call_tir(cls.NT_matmul6, (reshape63, lv81), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add91: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv38, gpt_neox_layers_3_attention_dense_bias3) layer_norm33: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add89, gpt_neox_layers_3_post_attention_layernorm_weight3, gpt_neox_layers_3_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv82 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv39 = R.call_tir(cls.NT_matmul7, (layer_norm33, lv82), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add92: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv39, gpt_neox_layers_3_mlp_dense_h_to_4h_bias3) gelu15: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add92) astype32: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu15, dtype="float16") lv83 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv40 = R.call_tir(cls.NT_matmul8, (astype32, lv83), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add93: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv40, gpt_neox_layers_3_mlp_dense_4h_to_h_bias3) astype33: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add93, dtype="float16") add94: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype33, add91) add95: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add94, add89) layer_norm34: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add95, gpt_neox_layers_4_input_layernorm_weight3, gpt_neox_layers_4_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv84 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight3, gpt_neox_layers_4_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv41 = R.call_tir(cls.NT_matmul5, (layer_norm34, lv84), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add96: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv41, gpt_neox_layers_4_attention_query_key_value_bias3) reshape64: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add96, R.shape([1, seq_len, 24, 64])) reshape65: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape64, R.shape([seq_len, 24, 64])) lv85 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape65), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape66: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv85, R.shape([1, seq_len, 8, 64])) reshape67: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape66, R.shape([1, seq_len, 512])) lv86 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight3, gpt_neox_layers_4_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv42 = R.call_tir(cls.NT_matmul6, (reshape67, lv86), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add97: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv42, gpt_neox_layers_4_attention_dense_bias3) layer_norm35: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add95, gpt_neox_layers_4_post_attention_layernorm_weight3, gpt_neox_layers_4_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv87 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv43 = R.call_tir(cls.NT_matmul7, (layer_norm35, lv87), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add98: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv43, gpt_neox_layers_4_mlp_dense_h_to_4h_bias3) gelu16: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add98) astype34: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu16, dtype="float16") lv88 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv44 = R.call_tir(cls.NT_matmul8, (astype34, lv88), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add99: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv44, gpt_neox_layers_4_mlp_dense_4h_to_h_bias3) astype35: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add99, dtype="float16") add100: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype35, add97) add101: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add100, add95) layer_norm36: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add101, gpt_neox_layers_5_input_layernorm_weight3, gpt_neox_layers_5_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv89 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight3, gpt_neox_layers_5_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv45 = R.call_tir(cls.NT_matmul5, (layer_norm36, lv89), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add102: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv45, gpt_neox_layers_5_attention_query_key_value_bias3) reshape68: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add102, R.shape([1, seq_len, 24, 64])) reshape69: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape68, R.shape([seq_len, 24, 64])) lv90 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape69), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape70: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv90, R.shape([1, seq_len, 8, 64])) reshape71: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape70, R.shape([1, seq_len, 512])) lv91 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight3, gpt_neox_layers_5_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv46 = R.call_tir(cls.NT_matmul6, (reshape71, lv91), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add103: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv46, gpt_neox_layers_5_attention_dense_bias3) layer_norm37: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add101, gpt_neox_layers_5_post_attention_layernorm_weight3, gpt_neox_layers_5_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv92 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv47 = R.call_tir(cls.NT_matmul7, (layer_norm37, lv92), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add104: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv47, gpt_neox_layers_5_mlp_dense_h_to_4h_bias3) gelu17: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add104) astype36: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu17, dtype="float16") lv93 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv48 = R.call_tir(cls.NT_matmul8, (astype36, lv93), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add105: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv48, gpt_neox_layers_5_mlp_dense_4h_to_h_bias3) astype37: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add105, dtype="float16") add106: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype37, add103) add107: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add106, add101) layer_norm38: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add107, gpt_neox_final_layer_norm_weight3, gpt_neox_final_layer_norm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) take1: R.Tensor((1, batch_size, 512), dtype="float16") = R.take(layer_norm38, logit_positions, axis=1) lv94 = R.call_tir(cls.dequantize, (embed_out_q_weight3, embed_out_q_scale3), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16")) lv49 = R.call_tir(cls.NT_matmul9, (take1, lv94), out_sinfo=R.Tensor((1, batch_size, vocab_size), dtype="float16")) astype38: R.Tensor((1, batch_size, vocab_size), dtype="float32") = R.astype(lv49, dtype="float32") gv3: R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype="float32"), R.Object) = astype38, paged_kv_cache R.output(gv3) return gv3 @R.function def batch_verify(input_embeds: R.Tensor((1, "seq_len", 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): gpt_neox_layers_0_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[3] gpt_neox_layers_0_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[5] gpt_neox_layers_0_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[8] gpt_neox_layers_0_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[11] gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[14] gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[17] gpt_neox_layers_1_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[19] gpt_neox_layers_1_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[21] gpt_neox_layers_1_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[24] gpt_neox_layers_1_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[27] gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[30] gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[33] gpt_neox_layers_2_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[35] gpt_neox_layers_2_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[37] gpt_neox_layers_2_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[40] gpt_neox_layers_2_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[43] gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[46] gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[49] gpt_neox_layers_3_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[51] gpt_neox_layers_3_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[53] gpt_neox_layers_3_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[56] gpt_neox_layers_3_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[59] gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[62] gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[65] gpt_neox_layers_4_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[67] gpt_neox_layers_4_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[69] gpt_neox_layers_4_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[72] gpt_neox_layers_4_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[75] gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[78] gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[81] gpt_neox_layers_5_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[83] gpt_neox_layers_5_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[85] gpt_neox_layers_5_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[88] gpt_neox_layers_5_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[91] gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[94] gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[97] gpt_neox_final_layer_norm_weight5: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias5: R.Tensor((512,), dtype="float16") = packed_params[99] embed_out_q_weight5: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale5: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] layer_norm52: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight5, gpt_neox_layers_0_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv126 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight5, gpt_neox_layers_0_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv50 = R.call_tir(cls.NT_matmul5, (layer_norm52, lv126), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add144: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv50, gpt_neox_layers_0_attention_query_key_value_bias5) reshape96: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add144, R.shape([1, seq_len, 24, 64])) reshape97: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape96, R.shape([seq_len, 24, 64])) lv127 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape97), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape98: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv127, R.shape([1, seq_len, 8, 64])) reshape99: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape98, R.shape([1, seq_len, 512])) lv128 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight5, gpt_neox_layers_0_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv51 = R.call_tir(cls.NT_matmul6, (reshape99, lv128), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add145: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv51, gpt_neox_layers_0_attention_dense_bias5) layer_norm53: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight5, gpt_neox_layers_0_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv129 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv52 = R.call_tir(cls.NT_matmul7, (layer_norm53, lv129), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add146: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv52, gpt_neox_layers_0_mlp_dense_h_to_4h_bias5) gelu24: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add146) astype52: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu24, dtype="float16") lv130 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv53 = R.call_tir(cls.NT_matmul8, (astype52, lv130), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add147: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv53, gpt_neox_layers_0_mlp_dense_4h_to_h_bias5) astype53: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add147, dtype="float16") add148: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype53, add145) add149: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add148, input_embeds) layer_norm54: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add149, gpt_neox_layers_1_input_layernorm_weight5, gpt_neox_layers_1_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv131 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight5, gpt_neox_layers_1_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv54 = R.call_tir(cls.NT_matmul5, (layer_norm54, lv131), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add150: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv54, gpt_neox_layers_1_attention_query_key_value_bias5) reshape100: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add150, R.shape([1, seq_len, 24, 64])) reshape101: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape100, R.shape([seq_len, 24, 64])) lv132 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape101), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape102: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv132, R.shape([1, seq_len, 8, 64])) reshape103: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape102, R.shape([1, seq_len, 512])) lv133 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight5, gpt_neox_layers_1_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv55 = R.call_tir(cls.NT_matmul6, (reshape103, lv133), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add151: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv55, gpt_neox_layers_1_attention_dense_bias5) layer_norm55: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add149, gpt_neox_layers_1_post_attention_layernorm_weight5, gpt_neox_layers_1_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv134 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv56 = R.call_tir(cls.NT_matmul7, (layer_norm55, lv134), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add152: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv56, gpt_neox_layers_1_mlp_dense_h_to_4h_bias5) gelu25: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add152) astype54: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu25, dtype="float16") lv135 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv57 = R.call_tir(cls.NT_matmul8, (astype54, lv135), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add153: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv57, gpt_neox_layers_1_mlp_dense_4h_to_h_bias5) astype55: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add153, dtype="float16") add154: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype55, add151) add155: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add154, add149) layer_norm56: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add155, gpt_neox_layers_2_input_layernorm_weight5, gpt_neox_layers_2_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv136 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight5, gpt_neox_layers_2_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv58 = R.call_tir(cls.NT_matmul5, (layer_norm56, lv136), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add156: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv58, gpt_neox_layers_2_attention_query_key_value_bias5) reshape104: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add156, R.shape([1, seq_len, 24, 64])) reshape105: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape104, R.shape([seq_len, 24, 64])) lv137 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape105), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape106: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv137, R.shape([1, seq_len, 8, 64])) reshape107: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape106, R.shape([1, seq_len, 512])) lv138 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight5, gpt_neox_layers_2_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv59 = R.call_tir(cls.NT_matmul6, (reshape107, lv138), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add157: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv59, gpt_neox_layers_2_attention_dense_bias5) layer_norm57: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add155, gpt_neox_layers_2_post_attention_layernorm_weight5, gpt_neox_layers_2_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv139 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv60 = R.call_tir(cls.NT_matmul7, (layer_norm57, lv139), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add158: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv60, gpt_neox_layers_2_mlp_dense_h_to_4h_bias5) gelu26: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add158) astype56: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu26, dtype="float16") lv140 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv61 = R.call_tir(cls.NT_matmul8, (astype56, lv140), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add159: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv61, gpt_neox_layers_2_mlp_dense_4h_to_h_bias5) astype57: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add159, dtype="float16") add160: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype57, add157) add161: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add160, add155) layer_norm58: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add161, gpt_neox_layers_3_input_layernorm_weight5, gpt_neox_layers_3_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv141 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight5, gpt_neox_layers_3_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv62 = R.call_tir(cls.NT_matmul5, (layer_norm58, lv141), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add162: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv62, gpt_neox_layers_3_attention_query_key_value_bias5) reshape108: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add162, R.shape([1, seq_len, 24, 64])) reshape109: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape108, R.shape([seq_len, 24, 64])) lv142 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape109), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape110: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv142, R.shape([1, seq_len, 8, 64])) reshape111: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape110, R.shape([1, seq_len, 512])) lv143 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight5, gpt_neox_layers_3_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv63 = R.call_tir(cls.NT_matmul6, (reshape111, lv143), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add163: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv63, gpt_neox_layers_3_attention_dense_bias5) layer_norm59: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add161, gpt_neox_layers_3_post_attention_layernorm_weight5, gpt_neox_layers_3_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv144 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv64 = R.call_tir(cls.NT_matmul7, (layer_norm59, lv144), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add164: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv64, gpt_neox_layers_3_mlp_dense_h_to_4h_bias5) gelu27: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add164) astype58: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu27, dtype="float16") lv145 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv65 = R.call_tir(cls.NT_matmul8, (astype58, lv145), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add165: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv65, gpt_neox_layers_3_mlp_dense_4h_to_h_bias5) astype59: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add165, dtype="float16") add166: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype59, add163) add167: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add166, add161) layer_norm60: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add167, gpt_neox_layers_4_input_layernorm_weight5, gpt_neox_layers_4_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv146 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight5, gpt_neox_layers_4_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv66 = R.call_tir(cls.NT_matmul5, (layer_norm60, lv146), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add168: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv66, gpt_neox_layers_4_attention_query_key_value_bias5) reshape112: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add168, R.shape([1, seq_len, 24, 64])) reshape113: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape112, R.shape([seq_len, 24, 64])) lv147 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape113), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape114: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv147, R.shape([1, seq_len, 8, 64])) reshape115: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape114, R.shape([1, seq_len, 512])) lv148 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight5, gpt_neox_layers_4_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv67 = R.call_tir(cls.NT_matmul6, (reshape115, lv148), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add169: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv67, gpt_neox_layers_4_attention_dense_bias5) layer_norm61: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add167, gpt_neox_layers_4_post_attention_layernorm_weight5, gpt_neox_layers_4_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv149 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv68 = R.call_tir(cls.NT_matmul7, (layer_norm61, lv149), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add170: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv68, gpt_neox_layers_4_mlp_dense_h_to_4h_bias5) gelu28: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add170) astype60: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu28, dtype="float16") lv150 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv69 = R.call_tir(cls.NT_matmul8, (astype60, lv150), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add171: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv69, gpt_neox_layers_4_mlp_dense_4h_to_h_bias5) astype61: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add171, dtype="float16") add172: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype61, add169) add173: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add172, add167) layer_norm62: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add173, gpt_neox_layers_5_input_layernorm_weight5, gpt_neox_layers_5_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv151 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight5, gpt_neox_layers_5_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv70 = R.call_tir(cls.NT_matmul5, (layer_norm62, lv151), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add174: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv70, gpt_neox_layers_5_attention_query_key_value_bias5) reshape116: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add174, R.shape([1, seq_len, 24, 64])) reshape117: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape116, R.shape([seq_len, 24, 64])) lv152 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape117), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape118: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv152, R.shape([1, seq_len, 8, 64])) reshape119: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape118, R.shape([1, seq_len, 512])) lv153 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight5, gpt_neox_layers_5_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv71 = R.call_tir(cls.NT_matmul6, (reshape119, lv153), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add175: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv71, gpt_neox_layers_5_attention_dense_bias5) layer_norm63: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add173, gpt_neox_layers_5_post_attention_layernorm_weight5, gpt_neox_layers_5_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv154 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv72 = R.call_tir(cls.NT_matmul7, (layer_norm63, lv154), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add176: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv72, gpt_neox_layers_5_mlp_dense_h_to_4h_bias5) gelu29: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add176) astype62: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu29, dtype="float16") lv155 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv73 = R.call_tir(cls.NT_matmul8, (astype62, lv155), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add177: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv73, gpt_neox_layers_5_mlp_dense_4h_to_h_bias5) astype63: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add177, dtype="float16") add178: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype63, add175) add179: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add178, add173) layer_norm64: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add179, gpt_neox_final_layer_norm_weight5, gpt_neox_final_layer_norm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv156 = R.call_tir(cls.dequantize, (embed_out_q_weight5, embed_out_q_scale5), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16")) lv74 = R.call_tir(cls.NT_matmul9, (layer_norm64, lv156), out_sinfo=R.Tensor((1, seq_len, vocab_size), dtype="float16")) astype64: R.Tensor((1, seq_len, vocab_size), dtype="float32") = R.astype(lv74, dtype="float32") gv5: R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype="float32"), R.Object) = astype64, paged_kv_cache R.output(gv5) return gv5 @R.function def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: max_batch_size = T.int64() max_total_seq_len = T.int64() prefill_chunk_size = T.int64() page_size = T.int64() support_sliding_window = T.int64() R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module gv: R.Tensor((), dtype="float16") = R.zeros(R.shape([]), dtype="float16") paged_kv_cache: R.Object = R.call_pure_packed("vm.builtin.paged_attention_kv_cache_create_reduced", R.shape([max_batch_size, max_total_seq_len, prefill_chunk_size, page_size, support_sliding_window]), R.shape([0, 6]), R.prim_value(8), R.prim_value(8), R.prim_value(64), R.prim_value(1), R.prim_value(1), R.prim_value(10000), gv, cls.tir_kv_cache_transpose_append, cls.batch_prefill_paged_kv, cls.batch_decode_paged_kv, cls.batch_prefill_paged_kv_sliding_window, cls.batch_decode_paged_kv_sliding_window, cls.batch_prefill_ragged_kv, cls.merge_state_inplace, cls.fused_rope, cls.copy_single_page, cls.tir_kv_cache_debug_get_kv, cls.compact_kv_copy, cls.batch_tree_attn, cls.tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) return paged_kv_cache @R.function def decode(input_embed: R.Tensor((1, 1, 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): gpt_neox_layers_0_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[3] gpt_neox_layers_0_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[5] gpt_neox_layers_0_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[8] gpt_neox_layers_0_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[11] gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[14] gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[17] gpt_neox_layers_1_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[19] gpt_neox_layers_1_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[21] gpt_neox_layers_1_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[24] gpt_neox_layers_1_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[27] gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[30] gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[33] gpt_neox_layers_2_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[35] gpt_neox_layers_2_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[37] gpt_neox_layers_2_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[40] gpt_neox_layers_2_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[43] gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[46] gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[49] gpt_neox_layers_3_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[51] gpt_neox_layers_3_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[53] gpt_neox_layers_3_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[56] gpt_neox_layers_3_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[59] gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[62] gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[65] gpt_neox_layers_4_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[67] gpt_neox_layers_4_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[69] gpt_neox_layers_4_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[72] gpt_neox_layers_4_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[75] gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[78] gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[81] gpt_neox_layers_5_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[83] gpt_neox_layers_5_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[85] gpt_neox_layers_5_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[88] gpt_neox_layers_5_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[91] gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[94] gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[97] gpt_neox_final_layer_norm_weight2: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias2: R.Tensor((512,), dtype="float16") = packed_params[99] embed_out_q_weight2: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale2: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] layer_norm13: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_input_layernorm_weight2, gpt_neox_layers_0_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv33 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight2, gpt_neox_layers_0_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv75 = R.call_tir(cls.NT_matmul10, (layer_norm13, lv33), out_sinfo=R.Tensor((1, 1, 1536), dtype="float16")) add36: R.Tensor((1, 1, 1536), dtype="float16") = R.add(lv75, gpt_neox_layers_0_attention_query_key_value_bias2) reshape24: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add36, R.shape([1, 1, 24, 64])) reshape25: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape24, R.shape([1, 24, 64])) lv34 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape25), out_sinfo=R.Tensor((1, 8, 64), dtype="float16")) reshape26: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv34, R.shape([1, 1, 8, 64])) reshape27: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape26, R.shape([1, 1, 512])) lv35 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight2, gpt_neox_layers_0_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv76 = R.call_tir(cls.NT_matmul11, (reshape27, lv35), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) add37: R.Tensor((1, 1, 512), dtype="float16") = R.add(lv76, gpt_neox_layers_0_attention_dense_bias2) layer_norm14: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight2, gpt_neox_layers_0_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv36 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv77 = R.call_tir(cls.NT_matmul12, (layer_norm14, lv36), out_sinfo=R.Tensor((1, 1, 2048), dtype="float32")) add38: R.Tensor((1, 1, 2048), dtype="float32") = R.add(lv77, gpt_neox_layers_0_mlp_dense_h_to_4h_bias2) gelu6: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add38) astype13: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu6, dtype="float16") lv37 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv78 = R.call_tir(cls.NT_matmul13, (astype13, lv37), out_sinfo=R.Tensor((1, 1, 512), dtype="float32")) add39: R.Tensor((1, 1, 512), dtype="float32") = R.add(lv78, gpt_neox_layers_0_mlp_dense_4h_to_h_bias2) astype14: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add39, dtype="float16") add40: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype14, add37) add41: R.Tensor((1, 1, 512), dtype="float16") = R.add(add40, input_embed) layer_norm15: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add41, gpt_neox_layers_1_input_layernorm_weight2, gpt_neox_layers_1_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv38 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight2, gpt_neox_layers_1_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv79 = R.call_tir(cls.NT_matmul10, (layer_norm15, lv38), out_sinfo=R.Tensor((1, 1, 1536), dtype="float16")) add42: R.Tensor((1, 1, 1536), dtype="float16") = R.add(lv79, gpt_neox_layers_1_attention_query_key_value_bias2) reshape28: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add42, R.shape([1, 1, 24, 64])) reshape29: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape28, R.shape([1, 24, 64])) lv39 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape29), out_sinfo=R.Tensor((1, 8, 64), dtype="float16")) reshape30: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv39, R.shape([1, 1, 8, 64])) reshape31: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape30, R.shape([1, 1, 512])) lv40 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight2, gpt_neox_layers_1_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv80 = R.call_tir(cls.NT_matmul11, (reshape31, lv40), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) add43: R.Tensor((1, 1, 512), dtype="float16") = R.add(lv80, gpt_neox_layers_1_attention_dense_bias2) layer_norm16: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add41, gpt_neox_layers_1_post_attention_layernorm_weight2, gpt_neox_layers_1_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv41 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv81 = R.call_tir(cls.NT_matmul12, (layer_norm16, lv41), out_sinfo=R.Tensor((1, 1, 2048), dtype="float32")) add44: R.Tensor((1, 1, 2048), dtype="float32") = R.add(lv81, gpt_neox_layers_1_mlp_dense_h_to_4h_bias2) gelu7: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add44) astype15: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu7, dtype="float16") lv42 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv82 = R.call_tir(cls.NT_matmul13, (astype15, lv42), out_sinfo=R.Tensor((1, 1, 512), dtype="float32")) add45: R.Tensor((1, 1, 512), dtype="float32") = R.add(lv82, gpt_neox_layers_1_mlp_dense_4h_to_h_bias2) astype16: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add45, dtype="float16") add46: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype16, add43) add47: R.Tensor((1, 1, 512), dtype="float16") = R.add(add46, add41) layer_norm17: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add47, gpt_neox_layers_2_input_layernorm_weight2, gpt_neox_layers_2_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv43 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight2, gpt_neox_layers_2_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv83 = R.call_tir(cls.NT_matmul10, (layer_norm17, lv43), out_sinfo=R.Tensor((1, 1, 1536), dtype="float16")) add48: R.Tensor((1, 1, 1536), dtype="float16") = R.add(lv83, gpt_neox_layers_2_attention_query_key_value_bias2) reshape32: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add48, R.shape([1, 1, 24, 64])) reshape33: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape32, R.shape([1, 24, 64])) lv44 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape33), out_sinfo=R.Tensor((1, 8, 64), dtype="float16")) reshape34: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv44, R.shape([1, 1, 8, 64])) reshape35: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape34, R.shape([1, 1, 512])) lv45 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight2, gpt_neox_layers_2_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv84 = R.call_tir(cls.NT_matmul11, (reshape35, lv45), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) add49: R.Tensor((1, 1, 512), dtype="float16") = R.add(lv84, gpt_neox_layers_2_attention_dense_bias2) layer_norm18: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add47, gpt_neox_layers_2_post_attention_layernorm_weight2, gpt_neox_layers_2_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv46 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv85 = R.call_tir(cls.NT_matmul12, (layer_norm18, lv46), out_sinfo=R.Tensor((1, 1, 2048), dtype="float32")) add50: R.Tensor((1, 1, 2048), dtype="float32") = R.add(lv85, gpt_neox_layers_2_mlp_dense_h_to_4h_bias2) gelu8: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add50) astype17: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu8, dtype="float16") lv47 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv86 = R.call_tir(cls.NT_matmul13, (astype17, lv47), out_sinfo=R.Tensor((1, 1, 512), dtype="float32")) add51: R.Tensor((1, 1, 512), dtype="float32") = R.add(lv86, gpt_neox_layers_2_mlp_dense_4h_to_h_bias2) astype18: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add51, dtype="float16") add52: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype18, add49) add53: R.Tensor((1, 1, 512), dtype="float16") = R.add(add52, add47) layer_norm19: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add53, gpt_neox_layers_3_input_layernorm_weight2, gpt_neox_layers_3_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv48 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight2, gpt_neox_layers_3_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv87 = R.call_tir(cls.NT_matmul10, (layer_norm19, lv48), out_sinfo=R.Tensor((1, 1, 1536), dtype="float16")) add54: R.Tensor((1, 1, 1536), dtype="float16") = R.add(lv87, gpt_neox_layers_3_attention_query_key_value_bias2) reshape36: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add54, R.shape([1, 1, 24, 64])) reshape37: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape36, R.shape([1, 24, 64])) lv49 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape37), out_sinfo=R.Tensor((1, 8, 64), dtype="float16")) reshape38: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv49, R.shape([1, 1, 8, 64])) reshape39: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape38, R.shape([1, 1, 512])) lv50 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight2, gpt_neox_layers_3_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv88 = R.call_tir(cls.NT_matmul11, (reshape39, lv50), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) add55: R.Tensor((1, 1, 512), dtype="float16") = R.add(lv88, gpt_neox_layers_3_attention_dense_bias2) layer_norm20: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add53, gpt_neox_layers_3_post_attention_layernorm_weight2, gpt_neox_layers_3_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv51 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv89 = R.call_tir(cls.NT_matmul12, (layer_norm20, lv51), out_sinfo=R.Tensor((1, 1, 2048), dtype="float32")) add56: R.Tensor((1, 1, 2048), dtype="float32") = R.add(lv89, gpt_neox_layers_3_mlp_dense_h_to_4h_bias2) gelu9: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add56) astype19: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu9, dtype="float16") lv52 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv90 = R.call_tir(cls.NT_matmul13, (astype19, lv52), out_sinfo=R.Tensor((1, 1, 512), dtype="float32")) add57: R.Tensor((1, 1, 512), dtype="float32") = R.add(lv90, gpt_neox_layers_3_mlp_dense_4h_to_h_bias2) astype20: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add57, dtype="float16") add58: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype20, add55) add59: R.Tensor((1, 1, 512), dtype="float16") = R.add(add58, add53) layer_norm21: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add59, gpt_neox_layers_4_input_layernorm_weight2, gpt_neox_layers_4_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv53 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight2, gpt_neox_layers_4_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv91 = R.call_tir(cls.NT_matmul10, (layer_norm21, lv53), out_sinfo=R.Tensor((1, 1, 1536), dtype="float16")) add60: R.Tensor((1, 1, 1536), dtype="float16") = R.add(lv91, gpt_neox_layers_4_attention_query_key_value_bias2) reshape40: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add60, R.shape([1, 1, 24, 64])) reshape41: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape40, R.shape([1, 24, 64])) lv54 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape41), out_sinfo=R.Tensor((1, 8, 64), dtype="float16")) reshape42: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv54, R.shape([1, 1, 8, 64])) reshape43: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape42, R.shape([1, 1, 512])) lv55 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight2, gpt_neox_layers_4_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv92 = R.call_tir(cls.NT_matmul11, (reshape43, lv55), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) add61: R.Tensor((1, 1, 512), dtype="float16") = R.add(lv92, gpt_neox_layers_4_attention_dense_bias2) layer_norm22: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add59, gpt_neox_layers_4_post_attention_layernorm_weight2, gpt_neox_layers_4_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv56 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv93 = R.call_tir(cls.NT_matmul12, (layer_norm22, lv56), out_sinfo=R.Tensor((1, 1, 2048), dtype="float32")) add62: R.Tensor((1, 1, 2048), dtype="float32") = R.add(lv93, gpt_neox_layers_4_mlp_dense_h_to_4h_bias2) gelu10: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add62) astype21: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu10, dtype="float16") lv57 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv94 = R.call_tir(cls.NT_matmul13, (astype21, lv57), out_sinfo=R.Tensor((1, 1, 512), dtype="float32")) add63: R.Tensor((1, 1, 512), dtype="float32") = R.add(lv94, gpt_neox_layers_4_mlp_dense_4h_to_h_bias2) astype22: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add63, dtype="float16") add64: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype22, add61) add65: R.Tensor((1, 1, 512), dtype="float16") = R.add(add64, add59) layer_norm23: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add65, gpt_neox_layers_5_input_layernorm_weight2, gpt_neox_layers_5_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv58 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight2, gpt_neox_layers_5_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv95 = R.call_tir(cls.NT_matmul10, (layer_norm23, lv58), out_sinfo=R.Tensor((1, 1, 1536), dtype="float16")) add66: R.Tensor((1, 1, 1536), dtype="float16") = R.add(lv95, gpt_neox_layers_5_attention_query_key_value_bias2) reshape44: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add66, R.shape([1, 1, 24, 64])) reshape45: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape44, R.shape([1, 24, 64])) lv59 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape45), out_sinfo=R.Tensor((1, 8, 64), dtype="float16")) reshape46: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv59, R.shape([1, 1, 8, 64])) reshape47: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape46, R.shape([1, 1, 512])) lv60 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight2, gpt_neox_layers_5_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv96 = R.call_tir(cls.NT_matmul11, (reshape47, lv60), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) add67: R.Tensor((1, 1, 512), dtype="float16") = R.add(lv96, gpt_neox_layers_5_attention_dense_bias2) layer_norm24: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add65, gpt_neox_layers_5_post_attention_layernorm_weight2, gpt_neox_layers_5_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv61 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv97 = R.call_tir(cls.NT_matmul12, (layer_norm24, lv61), out_sinfo=R.Tensor((1, 1, 2048), dtype="float32")) add68: R.Tensor((1, 1, 2048), dtype="float32") = R.add(lv97, gpt_neox_layers_5_mlp_dense_h_to_4h_bias2) gelu11: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add68) astype23: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu11, dtype="float16") lv62 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv98 = R.call_tir(cls.NT_matmul13, (astype23, lv62), out_sinfo=R.Tensor((1, 1, 512), dtype="float32")) add69: R.Tensor((1, 1, 512), dtype="float32") = R.add(lv98, gpt_neox_layers_5_mlp_dense_4h_to_h_bias2) astype24: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add69, dtype="float16") add70: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype24, add67) add71: R.Tensor((1, 1, 512), dtype="float16") = R.add(add70, add65) layer_norm25: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add71, gpt_neox_final_layer_norm_weight2, gpt_neox_final_layer_norm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv63 = R.call_tir(cls.dequantize, (embed_out_q_weight2, embed_out_q_scale2), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16")) lv99 = R.call_tir(cls.NT_matmul14, (layer_norm25, lv63), out_sinfo=R.Tensor((1, 1, vocab_size), dtype="float16")) astype25: R.Tensor((1, 1, vocab_size), dtype="float32") = R.astype(lv99, dtype="float32") gv2: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = astype25, paged_kv_cache R.output(gv2) return gv2 @R.function def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tensor(("seq_len", 512), dtype="float16"): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): gpt_neox_embed_in_q_weight: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0] gpt_neox_embed_in_q_scale: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1] lv = R.call_tir(cls.dequantize, (gpt_neox_embed_in_q_weight, gpt_neox_embed_in_q_scale), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16")) take: R.Tensor((seq_len, 512), dtype="float16") = R.take(lv, input_ids, axis=0) gv: R.Tensor((seq_len, 512), dtype="float16") = take R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul(lv95: R.Tensor((1536, 512), dtype="float16"), layer_norm39: R.Tensor(("batch_size", 1, 512), dtype="float16")) -> R.Tensor(("batch_size", 1, 1536), dtype="float16"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims75: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv95, axes=None) gv: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm39, permute_dims75, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul1(lv97: R.Tensor((512, 512), dtype="float16"), reshape75: R.Tensor(("batch_size", 1, 512), dtype="float16")) -> R.Tensor(("batch_size", 1, 512), dtype="float16"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims76: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv97, axes=None) gv: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape75, permute_dims76, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul10(lv33: R.Tensor((1536, 512), dtype="float16"), layer_norm13: R.Tensor((1, 1, 512), dtype="float16")) -> R.Tensor((1, 1, 1536), dtype="float16"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims25: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv33, axes=None) gv: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm13, permute_dims25, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul11(lv35: R.Tensor((512, 512), dtype="float16"), reshape27: R.Tensor((1, 1, 512), dtype="float16")) -> R.Tensor((1, 1, 512), dtype="float16"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims26: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv35, axes=None) gv: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape27, permute_dims26, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul12(lv36: R.Tensor((2048, 512), dtype="float16"), layer_norm14: R.Tensor((1, 1, 512), dtype="float16")) -> R.Tensor((1, 1, 2048), dtype="float32"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims27: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv36, axes=None) gv: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm14, permute_dims27, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul13(lv37: R.Tensor((512, 2048), dtype="float16"), astype13: R.Tensor((1, 1, 2048), dtype="float16")) -> R.Tensor((1, 1, 512), dtype="float32"): R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims28: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv37, axes=None) gv: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype13, permute_dims28, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul14(lv63: R.Tensor(("vocab_size", 512), dtype="float16"), layer_norm25: R.Tensor((1, 1, 512), dtype="float16")) -> R.Tensor((1, 1, "vocab_size"), dtype="float16"): vocab_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims49: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv63, axes=None) gv: R.Tensor((1, 1, vocab_size), dtype="float16") = R.matmul(layer_norm25, permute_dims49, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul2(lv98: R.Tensor((2048, 512), dtype="float16"), layer_norm40: R.Tensor(("batch_size", 1, 512), dtype="float16")) -> R.Tensor(("batch_size", 1, 2048), dtype="float32"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims77: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv98, axes=None) gv: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm40, permute_dims77, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul3(lv99: R.Tensor((512, 2048), dtype="float16"), astype39: R.Tensor(("batch_size", 1, 2048), dtype="float16")) -> R.Tensor(("batch_size", 1, 512), dtype="float32"): batch_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims78: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv99, axes=None) gv: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype39, permute_dims78, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul4(lv125: R.Tensor(("vocab_size", 512), dtype="float16"), layer_norm51: R.Tensor(("batch_size", 1, 512), dtype="float16")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float16"): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims99: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv125, axes=None) gv: R.Tensor((batch_size, 1, vocab_size), dtype="float16") = R.matmul(layer_norm51, permute_dims99, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul5(lv64: R.Tensor((1536, 512), dtype="float16"), layer_norm26: R.Tensor((1, "seq_len", 512), dtype="float16")) -> R.Tensor((1, "seq_len", 1536), dtype="float16"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims50: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv64, axes=None) gv: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm26, permute_dims50, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul6(lv66: R.Tensor((512, 512), dtype="float16"), reshape51: R.Tensor((1, "seq_len", 512), dtype="float16")) -> R.Tensor((1, "seq_len", 512), dtype="float16"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims51: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv66, axes=None) gv: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape51, permute_dims51, out_dtype="void") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul7(lv67: R.Tensor((2048, 512), dtype="float16"), layer_norm27: R.Tensor((1, "seq_len", 512), dtype="float16")) -> R.Tensor((1, "seq_len", 2048), dtype="float32"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims52: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv67, axes=None) gv: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm27, permute_dims52, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul8(lv68: R.Tensor((512, 2048), dtype="float16"), astype26: R.Tensor((1, "seq_len", 2048), dtype="float16")) -> R.Tensor((1, "seq_len", 512), dtype="float32"): seq_len = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims53: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv68, axes=None) gv: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype26, permute_dims53, out_dtype="float32") R.output(gv) return gv @R.function(private=True) def fused_relax_permute_dims_relax_matmul9(lv94: R.Tensor(("vocab_size", 512), dtype="float16"), take1: R.Tensor((1, "batch_size", 512), dtype="float16")) -> R.Tensor((1, "batch_size", "vocab_size"), dtype="float16"): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"Composite": "transpose_matmul_fuse", "Primitive": 1}) with R.dataflow(): permute_dims74: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv94, axes=None) gv: R.Tensor((1, batch_size, vocab_size), dtype="float16") = R.matmul(take1, permute_dims74, out_dtype="void") R.output(gv) return gv @R.function def prefill(input_embed: R.Tensor((1, "seq_len", 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): gpt_neox_layers_0_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[2] gpt_neox_layers_0_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[3] gpt_neox_layers_0_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[4] gpt_neox_layers_0_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[5] gpt_neox_layers_0_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[6] gpt_neox_layers_0_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[7] gpt_neox_layers_0_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[8] gpt_neox_layers_0_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[9] gpt_neox_layers_0_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[10] gpt_neox_layers_0_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[11] gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[12] gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[13] gpt_neox_layers_0_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[14] gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[15] gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[16] gpt_neox_layers_0_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[17] gpt_neox_layers_1_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[18] gpt_neox_layers_1_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[19] gpt_neox_layers_1_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[20] gpt_neox_layers_1_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[21] gpt_neox_layers_1_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[22] gpt_neox_layers_1_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[23] gpt_neox_layers_1_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[24] gpt_neox_layers_1_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[25] gpt_neox_layers_1_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[26] gpt_neox_layers_1_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[27] gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[28] gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[29] gpt_neox_layers_1_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[30] gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[31] gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[32] gpt_neox_layers_1_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[33] gpt_neox_layers_2_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[34] gpt_neox_layers_2_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[35] gpt_neox_layers_2_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[36] gpt_neox_layers_2_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[37] gpt_neox_layers_2_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[38] gpt_neox_layers_2_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[39] gpt_neox_layers_2_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[40] gpt_neox_layers_2_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[41] gpt_neox_layers_2_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[42] gpt_neox_layers_2_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[43] gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[44] gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[45] gpt_neox_layers_2_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[46] gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[47] gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[48] gpt_neox_layers_2_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[49] gpt_neox_layers_3_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[50] gpt_neox_layers_3_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[51] gpt_neox_layers_3_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[52] gpt_neox_layers_3_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[53] gpt_neox_layers_3_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[54] gpt_neox_layers_3_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[55] gpt_neox_layers_3_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[56] gpt_neox_layers_3_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[57] gpt_neox_layers_3_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[58] gpt_neox_layers_3_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[59] gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[60] gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[61] gpt_neox_layers_3_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[62] gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[63] gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[64] gpt_neox_layers_3_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[65] gpt_neox_layers_4_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[66] gpt_neox_layers_4_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[67] gpt_neox_layers_4_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[68] gpt_neox_layers_4_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[69] gpt_neox_layers_4_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[70] gpt_neox_layers_4_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[71] gpt_neox_layers_4_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[72] gpt_neox_layers_4_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[73] gpt_neox_layers_4_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[74] gpt_neox_layers_4_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[75] gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[76] gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[77] gpt_neox_layers_4_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[78] gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[79] gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[80] gpt_neox_layers_4_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[81] gpt_neox_layers_5_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[82] gpt_neox_layers_5_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[83] gpt_neox_layers_5_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[84] gpt_neox_layers_5_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[85] gpt_neox_layers_5_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[86] gpt_neox_layers_5_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[87] gpt_neox_layers_5_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[88] gpt_neox_layers_5_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[89] gpt_neox_layers_5_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[90] gpt_neox_layers_5_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[91] gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[92] gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[93] gpt_neox_layers_5_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[94] gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[95] gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[96] gpt_neox_layers_5_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[97] gpt_neox_final_layer_norm_weight1: R.Tensor((512,), dtype="float16") = packed_params[98] gpt_neox_final_layer_norm_bias1: R.Tensor((512,), dtype="float16") = packed_params[99] embed_out_q_weight1: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100] embed_out_q_scale1: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101] layer_norm: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_input_layernorm_weight1, gpt_neox_layers_0_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv1 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight1, gpt_neox_layers_0_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv100 = R.call_tir(cls.NT_matmul5, (layer_norm, lv1), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv100, gpt_neox_layers_0_attention_query_key_value_bias1) reshape: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add, R.shape([1, seq_len, 24, 64])) reshape1: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape, R.shape([seq_len, 24, 64])) lv2 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape2: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv2, R.shape([1, seq_len, 8, 64])) reshape3: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape2, R.shape([1, seq_len, 512])) lv3 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight1, gpt_neox_layers_0_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv101 = R.call_tir(cls.NT_matmul6, (reshape3, lv3), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add1: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv101, gpt_neox_layers_0_attention_dense_bias1) layer_norm1: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight1, gpt_neox_layers_0_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv4 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv102 = R.call_tir(cls.NT_matmul7, (layer_norm1, lv4), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add2: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv102, gpt_neox_layers_0_mlp_dense_h_to_4h_bias1) gelu: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add2) astype: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu, dtype="float16") lv5 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv103 = R.call_tir(cls.NT_matmul8, (astype, lv5), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add3: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv103, gpt_neox_layers_0_mlp_dense_4h_to_h_bias1) astype1: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add3, dtype="float16") add4: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype1, add1) add5: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add4, input_embed) layer_norm2: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add5, gpt_neox_layers_1_input_layernorm_weight1, gpt_neox_layers_1_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv6 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight1, gpt_neox_layers_1_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv104 = R.call_tir(cls.NT_matmul5, (layer_norm2, lv6), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add6: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv104, gpt_neox_layers_1_attention_query_key_value_bias1) reshape4: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add6, R.shape([1, seq_len, 24, 64])) reshape5: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape4, R.shape([seq_len, 24, 64])) lv7 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape6: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv7, R.shape([1, seq_len, 8, 64])) reshape7: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape6, R.shape([1, seq_len, 512])) lv8 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight1, gpt_neox_layers_1_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv105 = R.call_tir(cls.NT_matmul6, (reshape7, lv8), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add7: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv105, gpt_neox_layers_1_attention_dense_bias1) layer_norm3: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add5, gpt_neox_layers_1_post_attention_layernorm_weight1, gpt_neox_layers_1_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv9 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv106 = R.call_tir(cls.NT_matmul7, (layer_norm3, lv9), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add8: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv106, gpt_neox_layers_1_mlp_dense_h_to_4h_bias1) gelu1: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add8) astype2: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu1, dtype="float16") lv10 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv107 = R.call_tir(cls.NT_matmul8, (astype2, lv10), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add9: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv107, gpt_neox_layers_1_mlp_dense_4h_to_h_bias1) astype3: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add9, dtype="float16") add10: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype3, add7) add11: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add10, add5) layer_norm4: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add11, gpt_neox_layers_2_input_layernorm_weight1, gpt_neox_layers_2_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv11 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight1, gpt_neox_layers_2_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv108 = R.call_tir(cls.NT_matmul5, (layer_norm4, lv11), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add12: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv108, gpt_neox_layers_2_attention_query_key_value_bias1) reshape8: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add12, R.shape([1, seq_len, 24, 64])) reshape9: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape8, R.shape([seq_len, 24, 64])) lv12 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape10: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv12, R.shape([1, seq_len, 8, 64])) reshape11: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape10, R.shape([1, seq_len, 512])) lv13 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight1, gpt_neox_layers_2_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv109 = R.call_tir(cls.NT_matmul6, (reshape11, lv13), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add13: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv109, gpt_neox_layers_2_attention_dense_bias1) layer_norm5: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add11, gpt_neox_layers_2_post_attention_layernorm_weight1, gpt_neox_layers_2_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv14 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv110 = R.call_tir(cls.NT_matmul7, (layer_norm5, lv14), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add14: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv110, gpt_neox_layers_2_mlp_dense_h_to_4h_bias1) gelu2: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add14) astype4: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu2, dtype="float16") lv15 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv111 = R.call_tir(cls.NT_matmul8, (astype4, lv15), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add15: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv111, gpt_neox_layers_2_mlp_dense_4h_to_h_bias1) astype5: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add15, dtype="float16") add16: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype5, add13) add17: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add16, add11) layer_norm6: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add17, gpt_neox_layers_3_input_layernorm_weight1, gpt_neox_layers_3_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv16 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight1, gpt_neox_layers_3_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv112 = R.call_tir(cls.NT_matmul5, (layer_norm6, lv16), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add18: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv112, gpt_neox_layers_3_attention_query_key_value_bias1) reshape12: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add18, R.shape([1, seq_len, 24, 64])) reshape13: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape12, R.shape([seq_len, 24, 64])) lv17 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape14: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv17, R.shape([1, seq_len, 8, 64])) reshape15: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape14, R.shape([1, seq_len, 512])) lv18 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight1, gpt_neox_layers_3_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv113 = R.call_tir(cls.NT_matmul6, (reshape15, lv18), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add19: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv113, gpt_neox_layers_3_attention_dense_bias1) layer_norm7: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add17, gpt_neox_layers_3_post_attention_layernorm_weight1, gpt_neox_layers_3_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv19 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv114 = R.call_tir(cls.NT_matmul7, (layer_norm7, lv19), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add20: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv114, gpt_neox_layers_3_mlp_dense_h_to_4h_bias1) gelu3: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add20) astype6: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu3, dtype="float16") lv20 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv115 = R.call_tir(cls.NT_matmul8, (astype6, lv20), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add21: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv115, gpt_neox_layers_3_mlp_dense_4h_to_h_bias1) astype7: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add21, dtype="float16") add22: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype7, add19) add23: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add22, add17) layer_norm8: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add23, gpt_neox_layers_4_input_layernorm_weight1, gpt_neox_layers_4_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv21 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight1, gpt_neox_layers_4_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv116 = R.call_tir(cls.NT_matmul5, (layer_norm8, lv21), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add24: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv116, gpt_neox_layers_4_attention_query_key_value_bias1) reshape16: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add24, R.shape([1, seq_len, 24, 64])) reshape17: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape16, R.shape([seq_len, 24, 64])) lv22 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape18: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv22, R.shape([1, seq_len, 8, 64])) reshape19: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape18, R.shape([1, seq_len, 512])) lv23 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight1, gpt_neox_layers_4_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv117 = R.call_tir(cls.NT_matmul6, (reshape19, lv23), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add25: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv117, gpt_neox_layers_4_attention_dense_bias1) layer_norm9: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add23, gpt_neox_layers_4_post_attention_layernorm_weight1, gpt_neox_layers_4_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv24 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv118 = R.call_tir(cls.NT_matmul7, (layer_norm9, lv24), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add26: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv118, gpt_neox_layers_4_mlp_dense_h_to_4h_bias1) gelu4: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add26) astype8: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu4, dtype="float16") lv25 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv119 = R.call_tir(cls.NT_matmul8, (astype8, lv25), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add27: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv119, gpt_neox_layers_4_mlp_dense_4h_to_h_bias1) astype9: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add27, dtype="float16") add28: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype9, add25) add29: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add28, add23) layer_norm10: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add29, gpt_neox_layers_5_input_layernorm_weight1, gpt_neox_layers_5_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv26 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight1, gpt_neox_layers_5_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16")) lv120 = R.call_tir(cls.NT_matmul5, (layer_norm10, lv26), out_sinfo=R.Tensor((1, seq_len, 1536), dtype="float16")) add30: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(lv120, gpt_neox_layers_5_attention_query_key_value_bias1) reshape20: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add30, R.shape([1, seq_len, 24, 64])) reshape21: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape20, R.shape([seq_len, 24, 64])) lv27 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16")) reshape22: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv27, R.shape([1, seq_len, 8, 64])) reshape23: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape22, R.shape([1, seq_len, 512])) lv28 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight1, gpt_neox_layers_5_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16")) lv121 = R.call_tir(cls.NT_matmul6, (reshape23, lv28), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float16")) add31: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(lv121, gpt_neox_layers_5_attention_dense_bias1) layer_norm11: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add29, gpt_neox_layers_5_post_attention_layernorm_weight1, gpt_neox_layers_5_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv29 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16")) lv122 = R.call_tir(cls.NT_matmul7, (layer_norm11, lv29), out_sinfo=R.Tensor((1, seq_len, 2048), dtype="float32")) add32: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(lv122, gpt_neox_layers_5_mlp_dense_h_to_4h_bias1) gelu5: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add32) astype10: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu5, dtype="float16") lv30 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16")) lv123 = R.call_tir(cls.NT_matmul8, (astype10, lv30), out_sinfo=R.Tensor((1, seq_len, 512), dtype="float32")) add33: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(lv123, gpt_neox_layers_5_mlp_dense_4h_to_h_bias1) astype11: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add33, dtype="float16") add34: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype11, add31) add35: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add34, add29) layer_norm12: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add35, gpt_neox_final_layer_norm_weight1, gpt_neox_final_layer_norm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True) lv31 = R.call_tir(cls.index, (layer_norm12,), out_sinfo=R.Tensor((1, 1, 512), dtype="float16")) lv32 = R.call_tir(cls.dequantize, (embed_out_q_weight1, embed_out_q_scale1), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16")) lv124 = R.call_tir(cls.NT_matmul14, (lv31, lv32), out_sinfo=R.Tensor((1, 1, vocab_size), dtype="float16")) astype12: R.Tensor((1, 1, vocab_size), dtype="float32") = R.astype(lv124, dtype="float32") gv1: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = astype12, paged_kv_cache R.output(gv1) return gv1 @R.function def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64(is_size_var=True) vocab_size = T.int64(is_size_var=True) R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}}) cls = Module with R.dataflow(): lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", logits, R.shape([batch_size, vocab_size]), sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) lv1 = R.call_tir(cls.chunk_lse, (lv, temperature), out_sinfo=[R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32"), R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32")]) lv2: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[0] lv3: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[1] lv4 = R.call_tir(cls.softmax_with_chunked_sum, (lv, temperature, lv2, lv3), out_sinfo=R.Tensor((batch_size, vocab_size), dtype="float32")) gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", lv4, R.shape([batch_size, 1, vocab_size]), sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) R.output(gv) return gv
# from tvm.script import ir as I
# from tvm.script import tir as T
# from tvm.script import relax as R
@I.ir_module
class Module:
I.module_attrs({"system_lib_prefix": "gpt_neox_q4f16_1_"})
@T.prim_func(private=True)
def NT_matmul(var_layer_norm39: T.handle, lv95: T.Buffer((T.int64(1536), T.int64(512)), "float16"), var_NT_matmul: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
layer_norm39 = T.match_buffer(var_layer_norm39, (batch_size, T.int64(1), T.int64(512)), "float16")
NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(1536)), "float16")
# with T.block("root"):
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(1536), T.int64(512)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm39[v_i0, v_i1, v_k], lv95[v_i2, v_k])
T.writes(NT_matmul[v_i0, v_i1, v_i2])
with T.init():
NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm39[v_i0, v_i1, v_k] * lv95[v_i2, v_k]
@T.prim_func(private=True)
def NT_matmul1(var_reshape75: T.handle, lv97: T.Buffer((T.int64(512), T.int64(512)), "float16"), var_NT_matmul: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
reshape75 = T.match_buffer(var_reshape75, (batch_size, T.int64(1), T.int64(512)), "float16")
NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, T.int64(1), T.int64(512)), "float16")
# with T.block("root"):
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(512), T.int64(512)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(reshape75[v_i0, v_i1, v_k], lv97[v_i2, v_k])
T.writes(NT_matmul[v_i0, v_i1, v_i2])
with T.init():
NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape75[v_i0, v_i1, v_k] * lv97[v_i2, v_k]
@T.prim_func(private=True)
def NT_matmul10(layer_norm13: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), lv33: T.Buffer((T.int64(1536), T.int64(512)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(1536)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(1536), T.int64(512)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm13[v_i0, v_i1, v_k], lv33[v_i2, v_k])
T.writes(NT_matmul[v_i0, v_i1, v_i2])
with T.init():
NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + layer_norm13[v_i0, v_i1, v_k] * lv33[v_i2, v_k]
@T.prim_func(private=True)
def NT_matmul11(reshape27: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), lv35: T.Buffer((T.int64(512), T.int64(512)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(512), T.int64(512)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(reshape27[v_i0, v_i1, v_k], lv35[v_i2, v_k])
T.writes(NT_matmul[v_i0, v_i1, v_i2])
with T.init():
NT_matmul[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + reshape27[v_i0, v_i1, v_k] * lv35[v_i2, v_k]
@T.prim_func(private=True)
def NT_matmul12(layer_norm14: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), lv36: T.Buffer((T.int64(2048), T.int64(512)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float32")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(2048), T.int64(512)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm14[v_i0, v_i1, v_k], lv36[v_i2, v_k])
T.writes(NT_matmul[v_i0, v_i1, v_i2])
with T.init():
NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm14[v_i0, v_i1, v_k]) * T.Cast("float32", lv36[v_i2, v_k])
@T.prim_func(private=True)
def NT_matmul13(astype13: T.Buffer((T.int64(1), T.int64(1), T.int64(2048)), "float16"), lv37: T.Buffer((T.int64(512), T.int64(2048)), "float16"), NT_matmul: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float32")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(512), T.int64(2048)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(astype13[v_i0, v_i1, v_k], lv37[v_i2, v_k])
T.writes(NT_matmul[v_i0, v_i1, v_i2])
with T.init():
NT_matmul[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul[v_i0, v_i1, v_i2] = NT_matmul[v_i0, v_i1, v_i2] + T.Cast("float32", astype13[v_i0, v_i1, v_k]) * T.Cast("float32", lv37[v_i2, v_k])
@T.prim_func(private=True)
def NT_matmul14(layer_norm25: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16"), var_lv63: T.handle, var_NT_matmul: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
vocab_size = T.int64()
lv63 = T.match_buffer(var_lv63, (vocab_size, T.int64(512)), "float16")
NT_matmul = T.match_buffer(var_NT_matmul, (T.int64(1), T.int64(1), vocab_size), "float16")
# with T.block("root"):
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), vocab_size, T.int64(512)):
with T.block("NT_matmul"):